feat: screenshare

bots/bot_controller/bot_controller.py

@@ -1693,6 +1693,8 @@ def add_participant_event(self, event):
             webhook_trigger_type = WebhookTriggerTypes.PARTICIPANT_EVENTS_JOIN_LEAVE
         elif participant_event.event_type == ParticipantEventTypes.SPEECH_START or participant_event.event_type == ParticipantEventTypes.SPEECH_STOP:
             webhook_trigger_type = WebhookTriggerTypes.PARTICIPANT_EVENTS_SPEECH_START_STOP
+        elif participant_event.event_type == ParticipantEventTypes.SCREENSHARE_START or participant_event.event_type == ParticipantEventTypes.SCREENSHARE_STOP:
+            webhook_trigger_type = WebhookTriggerTypes.PARTICIPANT_EVENTS_SCREENSHARE_START_STOP
         else:
             logger.warning(f"Warning: Unknown participant event type: {participant_event.event_type}")
             return

bots/google_meet_bot_adapter/google_meet_chromedriver_payload.js

@@ -958,6 +958,8 @@ class UserManager {
         this.currentUsersMap = new Map();
         this.deviceOutputMap = new Map();
         this.currentUserId = null;
+        // Track the set of parentDeviceIds (real participant deviceIds) currently screen sharing
+        this.activeScreenSharerParentDeviceIds = new Set();
 
         this.ws = ws;
     }
@@ -1135,6 +1137,35 @@ class UserManager {
                 updatedUsers: updatedUsersToSend
             });
         }
+
+        this.detectScreenShareChanges();
+    }
+
+    detectScreenShareChanges() {
+        const currentScreenSharerParentDeviceIds = new Set(
+            this.getCurrentUsersInMeetingWhoAreScreenSharing().map(user => user.parentDeviceId)
+        );
+        for (const parentDeviceId of currentScreenSharerParentDeviceIds) {
+            if (!this.activeScreenSharerParentDeviceIds.has(parentDeviceId)) {
+                this.ws.sendJson({
+                    type: 'ScreenshareStartStopEvent',
+                    participantId: parentDeviceId,
+                    isScreenshareStart: true,
+                    timestamp: Date.now()
+                });
+            }
+        }
+        for (const parentDeviceId of this.activeScreenSharerParentDeviceIds) {
+            if (!currentScreenSharerParentDeviceIds.has(parentDeviceId)) {
+                this.ws.sendJson({
+                    type: 'ScreenshareStartStopEvent',
+                    participantId: parentDeviceId,
+                    isScreenshareStart: false,
+                    timestamp: Date.now()
+                });
+            }
+        }
+        this.activeScreenSharerParentDeviceIds = currentScreenSharerParentDeviceIds;
     }
 }
 

bots/migrations/0095_alter_participantevent_event_type_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.2.17 on 2026-10-13 22:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bots', '0094_remove_botevent_valid_event_type_event_sub_type_combinations_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='participantevent',
+            name='event_type',
+            field=models.IntegerField(choices=[(1, 'Join'), (2, 'Leave'), (3, 'Speech Start'), (4, 'Speech Stop'), (5, 'Update'), (6, 'Screenshare Start'), (7, 'Screenshare Stop')]),
+        ),
+        migrations.AlterField(
+            model_name='webhookdeliveryattempt',
+            name='webhook_trigger_type',
+            field=models.IntegerField(choices=[(1, 'Bot State Change'), (2, 'Transcript Update'), (3, 'Chat Messages Update'), (4, 'Participant Join/Leave'), (5, 'Calendar Events Update'), (6, 'Calendar State Change'), (7, 'Async Transcription State Change'), (8, 'Zoom OAuth Connection State Change'), (9, 'Bot Logs Update'), (10, 'Participant Speech Start/Stop'), (11, 'Participant Screenshare Start/Stop')], default=1),
+        ),
+    ]

bots/models.py

@@ -2262,6 +2262,8 @@ class ParticipantEventTypes(models.IntegerChoices):
     SPEECH_START = 3, "Speech Start"
     SPEECH_STOP = 4, "Speech Stop"
     UPDATE = 5, "Update"
+    SCREENSHARE_START = 6, "Screenshare Start"
+    SCREENSHARE_STOP = 7, "Screenshare Stop"
 
     @classmethod
     def type_to_api_code(cls, value):
@@ -2272,6 +2274,8 @@ def type_to_api_code(cls, value):
             cls.SPEECH_START: "speech_start",
             cls.SPEECH_STOP: "speech_stop",
             cls.UPDATE: "update",
+            cls.SCREENSHARE_START: "screenshare_start",
+            cls.SCREENSHARE_STOP: "screenshare_stop",
         }
         return mapping.get(value)
 
@@ -3206,6 +3210,7 @@ class WebhookTriggerTypes(models.IntegerChoices):
     ZOOM_OAUTH_CONNECTION_STATE_CHANGE = 8, "Zoom OAuth Connection State Change"
     BOT_LOGS_UPDATE = 9, "Bot Logs Update"
     PARTICIPANT_EVENTS_SPEECH_START_STOP = 10, "Participant Speech Start/Stop"
+    PARTICIPANT_EVENTS_SCREENSHARE_START_STOP = 11, "Participant Screenshare Start/Stop"
     # add other event types here
 
     @classmethod
@@ -3222,6 +3227,7 @@ def _get_mapping(cls):
             cls.ZOOM_OAUTH_CONNECTION_STATE_CHANGE: "zoom_oauth_connection.state_change",
             cls.BOT_LOGS_UPDATE: "bot_logs.update",
             cls.PARTICIPANT_EVENTS_SPEECH_START_STOP: "participant_events.speech_start_stop",
+            cls.PARTICIPANT_EVENTS_SCREENSHARE_START_STOP: "participant_events.screenshare_start_stop",
         }
 
     @classmethod

bots/projects_views.py

@@ -45,6 +45,7 @@
     CreditTransaction,
     InstanceHealthAlertsState,
     Participant,
+    ParticipantEvent,
     ParticipantEventTypes,
     Project,
     ProjectAccess,
@@ -936,7 +937,7 @@ def get(self, request, object_id, bot_object_id):
             chat_messages = obfuscate_chat_messages_for_bot_detail_view(chat_messages)
 
         # Get participants and participant events for this bot
-        participants = Participant.objects.filter(bot=bot, is_the_bot=False).prefetch_related("events").order_by("created_at")
+        participants = Participant.objects.filter(bot=bot, is_the_bot=False).prefetch_related(models.Prefetch("events", queryset=ParticipantEvent.objects.order_by("timestamp_ms", "id"))).order_by("created_at")
 
         # Get resource snapshots for this bot
         resource_snapshots = bot.resource_snapshots.all().order_by("created_at")

bots/teams_bot_adapter/teams_chromedriver_payload.js

@@ -1002,6 +1002,8 @@ class VirtualStreamToPhysicalStreamMappingManager {
 
         this.physicalClientStreamIdToVirtualStreamIdMapping = {}
         this.virtualStreamIdToPhysicalClientStreamIdMapping = {}
+        // Track participant IDs currently sharing their screen
+        this.activeScreenSharerParticipantIds = new Set();
     }
 
     getVirtualVideoStreamIdToSend() {
@@ -1094,13 +1096,44 @@ class VirtualStreamToPhysicalStreamMappingManager {
     upsertVirtualStream(virtualStream) {
         realConsole?.log('upsertVirtualStream', virtualStream, 'this.virtualStreams', this.virtualStreams);
         this.virtualStreams.set(virtualStream.sourceId.toString(), {...virtualStream, sourceId: virtualStream.sourceId.toString()});
+        this.detectScreenShareChanges();
     }
 
     removeVirtualStreamsForParticipant(participantId) {
         const virtualStreamsToRemove = Array.from(this.virtualStreams.values()).filter(virtualStream => virtualStream.participant.id === participantId);
         for (const virtualStream of virtualStreamsToRemove) {
             this.virtualStreams.delete(virtualStream.sourceId.toString());
         }
+        this.detectScreenShareChanges();
+    }
+
+    detectScreenShareChanges() {
+        const currentScreenSharerIds = new Set(
+            Array.from(this.virtualStreams.values())
+                .filter(vs => vs.isScreenShare && vs.isActive && vs.participant?.id)
+                .map(vs => vs.participant.id)
+        );
+        for (const participantId of currentScreenSharerIds) {
+            if (!this.activeScreenSharerParticipantIds.has(participantId)) {
+                window.ws?.sendJson({
+                    type: 'ScreenshareStartStopEvent',
+                    participantId: participantId,
+                    isScreenshareStart: true,
+                    timestamp: Date.now()
+                });
+            }
+        }
+        for (const participantId of this.activeScreenSharerParticipantIds) {
+            if (!currentScreenSharerIds.has(participantId)) {
+                window.ws?.sendJson({
+                    type: 'ScreenshareStartStopEvent',
+                    participantId: participantId,
+                    isScreenshareStart: false,
+                    timestamp: Date.now()
+                });
+            }
+        }
+        this.activeScreenSharerParticipantIds = currentScreenSharerIds;
     }
 
     upsertPhysicalClientStreamIdToVirtualStreamIdMapping(physicalClientStreamId, virtualStreamId) {

bots/templates/projects/project_bot_detail.html

@@ -507,13 +507,17 @@ <h2 class="accordion-header" id="heading{{ attempt.id }}">
                                     <div class="participant-events">
                                         <div class="participant-events-timeline mt-3">
                                             {% for event in participant.events.all %}
-                                                {% if event.event_type == ParticipantEventTypes.JOIN or event.event_type == ParticipantEventTypes.LEAVE %}
+                                                {% if event.event_type == ParticipantEventTypes.JOIN or event.event_type == ParticipantEventTypes.LEAVE or event.event_type == ParticipantEventTypes.SCREENSHARE_START or event.event_type == ParticipantEventTypes.SCREENSHARE_STOP %}
                                                     <div class="event-item d-flex align-items-start gap-3 py-2 border-bottom">
                                                         <div class="event-icon">
                                                             {% if event.event_type == ParticipantEventTypes.JOIN %}
                                                                 <i class="bi bi-box-arrow-in-right text-success"></i>
                                                             {% elif event.event_type == ParticipantEventTypes.LEAVE %}
                                                                 <i class="bi bi-box-arrow-left text-danger"></i>
+                                                            {% elif event.event_type == ParticipantEventTypes.SCREENSHARE_START %}
+                                                                <i class="bi bi-display text-primary"></i>
+                                                            {% elif event.event_type == ParticipantEventTypes.SCREENSHARE_STOP %}
+                                                                <i class="bi bi-display-fill text-secondary"></i>
                                                             {% endif %}
                                                         </div>
                                                         <div class="event-content flex-grow-1">
@@ -522,6 +526,10 @@ <h2 class="accordion-header" id="heading{{ attempt.id }}">
                                                                     Joined
                                                                 {% elif event.event_type == ParticipantEventTypes.LEAVE %}
                                                                     Left
+                                                                {% elif event.event_type == ParticipantEventTypes.SCREENSHARE_START %}
+                                                                    Screenshare Started
+                                                                {% elif event.event_type == ParticipantEventTypes.SCREENSHARE_STOP %}
+                                                                    Screenshare Stopped
                                                                 {% endif %}
                                                             </div>
                                                             <div class="event-timestamp text-muted small">{{ event.created_at|date:"M d, Y H:i:s" }}</div>

bots/tests/test_google_meet_bot_2.py

@@ -2489,3 +2489,170 @@ def test_could_not_join_allowed_when_leave_requested_before_bot_joined(self):
 
         self.bot.refresh_from_db()
         self.assertEqual(self.bot.state, BotStates.FATAL_ERROR)
+
+    @patch("bots.models.Bot.create_debug_recording", return_value=False)
+    @patch("bots.web_bot_adapter.web_bot_adapter.Display")
+    @patch("bots.web_bot_adapter.web_bot_adapter.webdriver.Chrome")
+    @patch("bots.bot_controller.bot_controller.AzureFileUploader")
+    @patch("bots.google_meet_bot_adapter.google_meet_ui_methods.GoogleMeetUIMethods.check_if_meeting_is_found", return_value=None)
+    @patch("bots.google_meet_bot_adapter.google_meet_ui_methods.GoogleMeetUIMethods.wait_for_host_if_needed", return_value=None)
+    @patch("time.time")
+    @patch("bots.tasks.deliver_webhook_task.deliver_webhook")
+    def test_bot_sends_screenshare_start_stop_participant_event_webhooks(
+        self,
+        mock_deliver_webhook,
+        mock_time,
+        mock_wait_for_host_if_needed,
+        mock_check_if_meeting_is_found,
+        MockFileUploader,
+        MockChromeDriver,
+        MockDisplay,
+        mock_create_debug_recording,
+    ):
+        mock_deliver_webhook.return_value = None
+
+        self.webhook_subscription = WebhookSubscription.objects.create(
+            project=self.project,
+            url="https://example.com/webhook",
+            triggers=[
+                WebhookTriggerTypes.BOT_STATE_CHANGE,
+                WebhookTriggerTypes.PARTICIPANT_EVENTS_JOIN_LEAVE,
+                WebhookTriggerTypes.PARTICIPANT_EVENTS_SCREENSHARE_START_STOP,
+            ],
+            is_active=True,
+        )
+
+        current_time = 1000.0
+        mock_time.return_value = current_time
+
+        self.recording.transcription_provider = TranscriptionProviders.CLOSED_CAPTION_FROM_PLATFORM
+        self.recording.save()
+
+        mock_uploader = create_mock_file_uploader()
+        MockFileUploader.return_value = mock_uploader
+
+        mock_driver = create_mock_google_meet_driver()
+        MockChromeDriver.return_value = mock_driver
+
+        mock_display = MagicMock()
+        MockDisplay.return_value = mock_display
+
+        controller = BotController(self.bot.id)
+
+        bot_thread = threading.Thread(target=controller.run)
+        bot_thread.daemon = True
+        bot_thread.start()
+
+        def simulate_join_flow():
+            nonlocal current_time
+
+            # Simulate participants joining
+            bot_participant_data = {"deviceId": "bot1", "fullName": "Test Bot", "active": True, "isCurrentUser": True}
+            controller.adapter.handle_participant_update(bot_participant_data)
+
+            participant_data = {"deviceId": "user1", "fullName": "Test User", "active": True, "isCurrentUser": False}
+            controller.adapter.handle_participant_update(participant_data)
+
+            controller.adapter.last_audio_message_processed_time = current_time
+
+            time.sleep(3)
+
+            # Simulate screenshare start event for the user participant
+            controller.adapter.handle_screenshare_start_stop_event(
+                {
+                    "participantId": "user1",
+                    "isScreenshareStart": True,
+                    "timestamp": int(current_time * 1000),
+                }
+            )
+
+            time.sleep(0.5)
+
+            # Simulate screenshare stop event for the user participant
+            controller.adapter.handle_screenshare_start_stop_event(
+                {
+                    "participantId": "user1",
+                    "isScreenshareStart": False,
+                    "timestamp": int(current_time * 1000) + 30000,
+                }
+            )
+
+            # Also simulate a screenshare event for the bot (should NOT produce a webhook)
+            controller.adapter.handle_screenshare_start_stop_event(
+                {
+                    "participantId": "bot1",
+                    "isScreenshareStart": True,
+                    "timestamp": int(current_time * 1000) + 31000,
+                }
+            )
+
+            time.sleep(1)
+
+            # Simulate participant leaving
+            participant_data = {"deviceId": "user1", "fullName": "Test User", "active": False, "isCurrentUser": False}
+            controller.adapter.handle_participant_update(participant_data)
+
+            # Trigger auto-leave
+            controller.adapter.only_one_participant_in_meeting_at = time.time() - 10000000000
+            time.sleep(4)
+
+            connection.close()
+
+        threading.Timer(2, simulate_join_flow).start()
+
+        bot_thread.join(timeout=15)
+
+        self.bot.refresh_from_db()
+        self.assertEqual(self.bot.state, BotStates.ENDED)
+
+        # Verify ParticipantEvent records for screenshare start/stop were created for the user
+        user_participant_events = ParticipantEvent.objects.filter(participant__bot=self.bot, participant__uuid="user1")
+        screenshare_start_event = user_participant_events.filter(event_type=ParticipantEventTypes.SCREENSHARE_START).first()
+        self.assertIsNotNone(screenshare_start_event, "Expected a SCREENSHARE_START participant event for the user")
+        self.assertEqual(screenshare_start_event.participant.full_name, "Test User")
+        self.assertEqual(screenshare_start_event.timestamp_ms, int(current_time * 1000))
+
+        screenshare_stop_event = user_participant_events.filter(event_type=ParticipantEventTypes.SCREENSHARE_STOP).first()
+        self.assertIsNotNone(screenshare_stop_event, "Expected a SCREENSHARE_STOP participant event for the user")
+        self.assertEqual(screenshare_stop_event.participant.full_name, "Test User")
+        self.assertEqual(screenshare_stop_event.timestamp_ms, int(current_time * 1000) + 30000)
+
+        # Verify that a SCREENSHARE_START event was also created for the bot participant
+        bot_screenshare_events = ParticipantEvent.objects.filter(participant__bot=self.bot, participant__uuid="bot1", event_type=ParticipantEventTypes.SCREENSHARE_START)
+        self.assertEqual(bot_screenshare_events.count(), 1, "Expected a SCREENSHARE_START event for the bot participant")
+
+        # Verify webhook delivery attempts for screenshare start/stop
+        screenshare_webhook_attempts = WebhookDeliveryAttempt.objects.filter(
+            bot=self.bot,
+            webhook_trigger_type=WebhookTriggerTypes.PARTICIPANT_EVENTS_SCREENSHARE_START_STOP,
+        )
+        # Only user events should trigger webhooks (bot events are suppressed)
+        self.assertEqual(screenshare_webhook_attempts.count(), 2, "Expected exactly 2 screenshare webhook delivery attempts (screenshare_start and screenshare_stop for the user)")
+
+        screenshare_start_webhook = screenshare_webhook_attempts.filter(payload__event_type="screenshare_start").first()
+        self.assertIsNotNone(screenshare_start_webhook)
+        self.assertEqual(screenshare_start_webhook.payload["participant_name"], "Test User")
+        self.assertEqual(screenshare_start_webhook.payload["participant_uuid"], "user1")
+        self.assertEqual(screenshare_start_webhook.payload["timestamp_ms"], int(current_time * 1000))
+
+        screenshare_stop_webhook = screenshare_webhook_attempts.filter(payload__event_type="screenshare_stop").first()
+        self.assertIsNotNone(screenshare_stop_webhook)
+        self.assertEqual(screenshare_stop_webhook.payload["participant_name"], "Test User")
+        self.assertEqual(screenshare_stop_webhook.payload["participant_uuid"], "user1")
+        self.assertEqual(screenshare_stop_webhook.payload["timestamp_ms"], int(current_time * 1000) + 30000)
+
+        # Verify that no screenshare webhook was created for the bot participant
+        bot_screenshare_webhooks = screenshare_webhook_attempts.filter(payload__participant_uuid="bot1")
+        self.assertEqual(bot_screenshare_webhooks.count(), 0, "Expected no screenshare webhooks for the bot participant")
+
+        # Verify join/leave webhooks were also created (ensuring screenshare events don't interfere)
+        join_leave_webhook_attempts = WebhookDeliveryAttempt.objects.filter(
+            bot=self.bot,
+            webhook_trigger_type=WebhookTriggerTypes.PARTICIPANT_EVENTS_JOIN_LEAVE,
+        )
+        self.assertGreater(join_leave_webhook_attempts.count(), 0, "Expected join/leave webhook delivery attempts")
+
+        controller.cleanup()
+        bot_thread.join(timeout=5)
+
+        connection.close()