adding optimizations for messages with many responses

also added a few UI fixes

CHANGELOG.MD

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.3.1-rc29] - 2024-12-10
+
+### Changed
+
+- Updated the processing for agent messages to minimize memory bloat
+  - Updated file download process to cache filemeta information and open file descriptors to minimize disk/database access
+  - Updated agent response processing to cache task data for one update instead of per-response
+- Updated MythicRPCResponseCreate to also touch the timestamp of the corresponding task to update the UI streaming
+
 ## [3.3.1-rc28] - 2024-11-26
 
 ### Changed

MythicReactUI/CHANGELOG.MD

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.63] - 2024-12-10
+
+### Changed
+
+- Updated UI to better handle pagination and streaming task responses
+- Updated UI to fix bug with submitting payload description update
+- Updated UI to fix bug with editing prior modal tasking via command line
+
 ## [0.2.62] - 2024-12-04
 
 ### Changed

MythicReactUI/src/components/MythicComponents/MythicDialog.js

@@ -62,6 +62,9 @@ export function MythicModifyStringDialog(props) {
   const theme = useTheme();
     const onCommitSubmit = () => {
         props.onSubmit(comment);
+        if(props.dontCloseOnSubmit){
+            return;
+        }
         props.onClose();
     }
     const onChange = (value) => {

MythicReactUI/src/components/pages/Callbacks/CallbacksTabsTaskingInput.js

@@ -1333,7 +1333,8 @@ export function CallbacksTabsTaskingInputPreMemo(props){
         let cmdGroupName = ["Default"];
         let parsedWithPositionalParameters = {};
         let params = splitMessage.slice(1).join(" ");
-        if(unmodifiedHistoryValue.includes("modal") || unmodifiedHistoryValue.includes("browserscript")){
+        let failed_json_parse = true;
+        if(unmodifiedHistoryValue === "modal" || unmodifiedHistoryValue.includes("browserscript")){
             // these are the two kinds that'll introduce dictionary values as original_params
             try{
                 parsedWithPositionalParameters = JSON.parse(params);
@@ -1344,12 +1345,13 @@ export function CallbacksTabsTaskingInputPreMemo(props){
                     snackActions.warning("Two or more of the specified parameters can't be used together", snackMessageStyles);
                     return;
                 }
+                failed_json_parse = false;
 
             }catch(error){
-                snackActions.warning("Failed to parse modified JSON value", snackMessageStyles);
-                return;
+                failed_json_parse = true;
             }
-        }else{
+        }
+        if(failed_json_parse){
             let parsed = parseCommandLine(params, cmd);
             //console.log("result of parseCommandLine", parsed, !parsed)
             if(parsed === undefined){

MythicReactUI/src/components/pages/Callbacks/ResponseDisplay.js

@@ -172,7 +172,7 @@ const NonInteractiveResponseDisplay = (props) => {
     setOpenBackdrop(true);
     onSubmitPageChange(1);
   }, [props.task.id]);
-  const subscriptionDataCallback = React.useCallback( ({data}) => {
+  const subscriptionDataCallback =  ({data}) => {
     //console.log("fetchLimit", fetchLimit, "totalCount", totalCount);
     if(props.task.id !== taskID.current){
       console.log("props.task.id !== taskID.current", props.task.id, taskID.current)
@@ -191,10 +191,12 @@ const NonInteractiveResponseDisplay = (props) => {
       setTotalCount(responseArray.length);
       setOpenBackdrop(false);
     } else {
-      if(totalCount >= initialResponseStreamLimit && initialResponseStreamLimit > 0){
+
+      if(rawResponses.length >= initialResponseStreamLimit && initialResponseStreamLimit > 0 && !props.selectAllOutput){
         // we won't display it
         console.log("got more than we can see currently", totalCount);
         setOpenBackdrop(false);
+        setTotalCount(totalCount + data.data.response_stream.length);
         return;
       }
       // we still have some room to view more, but only room for initialResponseStreamLimit - totalFetched.current
@@ -208,8 +210,9 @@ const NonInteractiveResponseDisplay = (props) => {
       }, rawResponses);
       // sort them to make sure we're still in order
       newerResponses.sort( (a,b) => a.id > b.id ? 1 : -1);
+      setTotalCount(totalCount + data.data.response_stream.length);
       // newerResponses is everything we've seen plus everything new
-      if(initialResponseStreamLimit > 0){
+      if(initialResponseStreamLimit > 0 && !props.selectAllOutput){
         // take just the responses that make up our stream limit
         const finalRawResponses = newerResponses.slice(0, initialResponseStreamLimit);
         const outputResponses = finalRawResponses.reduce( (prev, cur) => {
@@ -228,7 +231,7 @@ const NonInteractiveResponseDisplay = (props) => {
       }
       setOpenBackdrop(false);
     }
-  }, [output, rawResponses, totalCount, props.task.id]);
+  };
   useSubscription(subResponsesStream, {
     variables: {task_id: props.task.id},
     fetchPolicy: "network_only",

MythicReactUI/src/components/pages/Callbacks/ResponseDisplayTable.js

@@ -39,6 +39,7 @@ export const getIconName = (iconName) => {
     case "diskimage":
       return faBoxOpen;
     case "executable":
+    case "cog":
       return faCog;
     case "word":
       return faFileWord;

MythicReactUI/src/components/pages/Callbacks/TaskCommentDialog.js

@@ -38,6 +38,7 @@ export function TaskCommentDialog(props) {
     const [updateComment] = useMutation(updateCommentMutation, {
         update: (cache, {data}) => {
             //console.log(data);
+            props.onClose();
         }
     });
     if (loading) {
@@ -49,14 +50,14 @@ export function TaskCommentDialog(props) {
     }
     const onCommitSubmit = (newValue) => {
         updateComment({variables: {task_id: props.task_id, comment: newValue}});
-        props.onClose();
     }
 
   return (
     <React.Fragment>
         <MythicModifyStringDialog title={`Edit Task Comment`}
                                   onClose={props.onClose}
                                   value={comment}
+                                  dontCloseOnSubmit={true}
                                   onSubmit={onCommitSubmit} />
   </React.Fragment>
   );

MythicReactUI/src/components/pages/PayloadTypesC2Profiles/C2ProfilesCard.js

@@ -179,7 +179,7 @@ export function C2ProfilesRow({service, showDeleted}) {
                     <Typography variant="body1" component="p">
                         <b>Supported Agents:</b> {service.payloadtypec2profiles.filter( (pt) => !pt.payloadtype.deleted ).map(c => c.payloadtype.name).join(", ")}
                     </Typography>
-                    <Typography variant="body2" component="p">
+                    <Typography variant="body2" component="p" style={{whiteSpace: "pre-wrap"}}>
                         <b>Description: </b>{service.description}
                     </Typography>
                 </MythicTableCell>

MythicReactUI/src/components/pages/PayloadTypesC2Profiles/PayloadTypeCard.js

@@ -102,8 +102,8 @@ export function PayloadTypeRow({service, showDeleted}){
                             <b>Wrapped Payload Types:</b> {wrappedPayloads}
                         </Typography>
                     )}
-                    <Typography variant="body2" component="p">
-                        <b>Description:</b><br/>{service.note}
+                    <Typography variant="body2" component="p" style={{whiteSpace: "pre-wrap"}}>
+                        <b>Description: </b>{service.note}
                     </Typography>
                 </MythicTableCell>
                 <MythicTableCell>

MythicReactUI/src/components/pages/PayloadTypesC2Profiles/TranslationContainerCard.js

@@ -90,7 +90,7 @@ export function TranslationContainerRow({service, showDeleted}) {
                 <Typography variant="body1" component="p">
                     <b>Supported Agents:</b> {service.payloadtypes.filter(pt => !pt.deleted).map( (pt) => pt.name).join(", ")}
                 </Typography>
-                <Typography variant="body2" component="p">
+                <Typography variant="body2" component="p" style={{whiteSpace: "pre-wrap"}}>
                     <b>Description: </b>{service.description}
                 </Typography>
             </MythicTableCell>

MythicReactUI/src/components/pages/Payloads/PayloadDescriptionDialog.js

@@ -47,7 +47,11 @@ export function PayloadDescriptionDialog(props) {
     });
     const [updateDescription] = useMutation(updateDescriptionMutation, {
         onCompleted: (data) => {
-            snackActions.success("Updated Payload Description")
+            snackActions.success("Updated Payload Description");
+            props.onClose();
+        },
+        onError: (data) => {
+            console.log(data);
         }
     });
     const [updateCallbackDescriptions] = useMutation(updateCallbackDescriptionsMutation, {
@@ -86,7 +90,6 @@ export function PayloadDescriptionDialog(props) {
     const updatePayloadDescription = () => {
         setOpenUpdateAll(false);
         updateDescription({variables: {payload_id: props.payload_id, description: description.current}});
-        props.onClose();
     }
     const onCommitSubmit = (updatedMessage) => {
         description.current = updatedMessage;
@@ -103,7 +106,8 @@ export function PayloadDescriptionDialog(props) {
         <MythicModifyStringDialog title={"Edit Payload Description"}
                                   maxRows={5}
                                   onClose={props.onClose}
-                                  value={description.current}
+                                  value={oldDescription}
+                                  dontCloseOnSubmit={true}
                                   onSubmit={onCommitSubmit} />
         {openUpdateAll &&
             <MythicConfirmDialog title={"Update Associated Callback's Descriptions?"}

MythicReactUI/src/components/pages/Payloads/PayloadFilenameDialog.js

@@ -39,6 +39,7 @@ export function PayloadFilenameDialog(props) {
     const [updateDescription] = useMutation(updateDescriptionMutation, {
         onCompleted: (data) => {
           snackActions.success("Updated filename");
+          props.onClose();
         }
     });
     if (loading) {
@@ -50,7 +51,7 @@ export function PayloadFilenameDialog(props) {
     }
     const onCommitSubmit = (newDescription) => {
         updateDescription({variables: {file_id: fileId, filename: newDescription}});
-        props.onClose();
+
     }
 
   return (
@@ -59,6 +60,7 @@ export function PayloadFilenameDialog(props) {
                                   maxRows={2}
                                   onClose={props.onClose}
                                   value={description}
+                                  dontCloseOnSubmit={true}
                                   onSubmit={onCommitSubmit} />
   </React.Fragment>
   );

MythicReactUI/src/components/pages/Settings/SettingsOperatorExperimentalUIConfigDialog.js

@@ -19,9 +19,9 @@ import {snackActions} from "../../utilities/Snackbar";
 export function SettingsOperatorExperimentalUIConfigDialog(props) {
     const initialResponseStreamLimit = GetMythicSetting({setting_name: "experiment-responseStreamLimit", default_value: 50})
     const [newResponseStreamLimit, setNewResponseStreamLimit] = React.useState(initialResponseStreamLimit);
-    const [updateSetting] = useSetMythicSetting();
+    const [updateSetting, _] = useSetMythicSetting();
     const onNewResponseStreamLimitChange = (name, value, error) => {
-        setNewResponseStreamLimit(value);
+        setNewResponseStreamLimit(parseInt(value));
     }
 
     const onAccept = () => {

MythicReactUI/src/components/pages/Settings/SettingsOperatorTable.js

@@ -56,7 +56,7 @@ export function SettingsOperatorTable(props){
         props.onNewBot(username);
         setOpenNewBotDialog(false);
     }
-    const userData = props.operators.filter(o => o.id === (props.me?.user?.id || 0))
+    const userData = props.operators.filter(o => o.id === props.me?.user?.id)
     const userIsAdmin = userData.length > 0 ? userData[0].admin : false;
     const [showDeleted, setShowDeleted] = React.useState(false);
     const [openGlobalSettingsDialog, setOpenGlobalSettingsDialog] = React.useState(false);

MythicReactUI/src/index.js

@@ -14,7 +14,7 @@ import {snackActions} from './components/utilities/Snackbar';
 import jwt_decode from 'jwt-decode';
 import {meState} from './cache';
 
-export const mythicUIVersion = "0.2.62";
+export const mythicUIVersion = "0.2.63";
 
 let fetchingNewToken = false;
 

Mythic_CLI/Changelog.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## 0.3.8 - 2024-12-10 
+
+### Changed
+
+- Added check for docker compose plugin, not just docker version
+
 ## 0.3.7 - 
 
 ### Changed

Mythic_CLI/src/cmd/config/vars.go

@@ -4,5 +4,5 @@ package config
 
 var (
 	// Version Mythic CLI version
-	Version = "v0.3.7"
+	Version = "v0.3.8"
 )

Mythic_CLI/src/cmd/manager/dockerComposeManager.go

@@ -262,7 +262,16 @@ func (d *DockerComposeManager) CheckRequiredManagerVersion() bool {
 		return false
 	}
 	if semver.Compare("v"+outputString, "v20.10.22") >= 0 {
-		return true
+		composeCheckString, err := d.runDocker([]string{"compose", "version"})
+		if err != nil {
+			log.Printf("[-] Failed to get docker compose: %v\n", err)
+			return false
+		}
+		if strings.Contains(composeCheckString, "Docker Compose version v") {
+			return true
+		}
+		log.Printf("[-] Unable to find compose plugin. Please install the docker compose plugin.\n")
+		return false
 	}
 	log.Printf("[-] Docker version is too old, %s, for Mythic. Please update\n", outputString)
 	return false

VERSION

@@ -1 +1 @@
-3.3.1-rc28
+3.3.1-rc29

mythic-docker/.docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21 AS builder
+FROM golang:1.23 AS builder
 
 WORKDIR /usr/src/app
 

mythic-docker/Dockerfile

@@ -1,6 +1,6 @@
 #FROM ghcr.io/its-a-feature/mythic_server:v0.0.3.8
 
-FROM golang:1.21 AS builder
+FROM golang:1.23 AS builder
 
 WORKDIR /usr/src/app
 

mythic-docker/src/Makefile

@@ -1,5 +1,5 @@
 BINARY_NAME?=mythic_server
-JWT_SECRET?=
+JWT_SECRET?="2Zd2rGaAhTnuGVEsq9LeYVFXns5PN1"
 POSTGRES_HOST?="127.0.0.1"
 POSTGRES_PASSWORD?="0zoHFSTbiT7AzJvnbvQ5Lt65YkIsuT"
 DEBUG_LEVEL?="debug"
@@ -13,6 +13,7 @@ MYTHIC_ADMIN_USERNAME?="mythic_admin"
 MYTHIC_ADMIN_PASSWORD?="mythic_password"
 MYTHIC_DEFAULT_OPERATION?="Operation Chimera"
 MYTHIC_DEBUG_AGENT_MESSAGE?=false
+MYTHIC_DOCKER_NETWORKING?="host"
 
 export
 
@@ -39,6 +40,7 @@ MYTHIC_DEFAULT_OPERATION=${MYTHIC_DEFAULT_OPERATION} \
 MYTHIC_SERVER_BIND_LOCALHOST_ONLY=${MYTHIC_SERVER_BIND_LOCALHOST_ONLY} \
 MYTHIC_SERVER_DYNAMIC_PORTS=${MYTHIC_SERVER_DYNAMIC_PORTS} \
 MYTHIC_SERVER_GRPC_PORT=${MYTHIC_SERVER_GRPC_PORT} \
+MYTHIC_DOCKER_NETWORKING=${MYTHIC_DOCKER_NETWORKING} \
 ./${BINARY_NAME}
 
 run_local_build:
@@ -61,7 +63,7 @@ build_protobuf: build_protobuf_go
 build_and_build_protobuf: build_protobuf build
 build_final: build copy_to_final
 build_and_run: build_and_build_protobuf run_local_build
-local_testing: build run_custom_values
+run_custom: build run_custom_values
 
 clean:
 	go clean

mythic-docker/src/VERSION

@@ -1 +1 @@
-3.3.1-rc28
+3.3.1-rc29