diff --git a/docs/train/train-on-cloud/static/image/workflow/.DS_Store b/docs/train/train-on-cloud/static/image/workflow/.DS_Store deleted file mode 100644 index 02d219d..0000000 Binary files a/docs/train/train-on-cloud/static/image/workflow/.DS_Store and /dev/null differ diff --git a/docs/train/workflow.md b/docs/workflow/compound-ai-workflows/index.md similarity index 79% rename from docs/train/workflow.md rename to docs/workflow/compound-ai-workflows/index.md index af92b18..eaf9b47 100644 --- a/docs/train/workflow.md +++ b/docs/workflow/compound-ai-workflows/index.md @@ -1,15 +1,11 @@ --- -sidebar_position: 5 +sidebar_position: 2 --- -# Workflow for Compound Training Jobs +# Compound AI Workflows -As GenAI advances, there arises a frequent requirement to merge model training or deployment tasks into cohesive pipelines. Common instances involve: -1. Developing a model training pipeline that collects data, preprocesses it followed by training and evalution. -2. Compound AI systems tying various model deployment endpoints. -This is where the FedML Workflow API becomes advantageous. The Workflow API offers an intuitive platform for outlining tasks and their interdependencies, utilizing the underlying FedML core suite of APIs. -![ ](./train-on-cloud/static/image/workflow/1_workflow.png) +![ ](./static/image/1_workflow.png) The execution order of the above example workflow will be as follows: @@ -94,8 +90,8 @@ The journey for users with this workflow goes like this: - Training takes the data and trains a model, saving its progress along the way. - Next, the evaluation job kicks in. It grabs the trained model, evaluates it and returns the evaluation results as the final workflow output -![ ](./train-on-cloud/static/image/workflow/2_journey.png) +![ ](./static/image/2_journey.png) Here's another visual representation of a workflow scenario where two model endpoints are connected. The process begins when the end user interacts with the inference endpoint by providing a prompt. This prompt is initially processed by model 1, and the resulting output serves as the input for another model. Finally, the output from this second model is returned to the user as the final result. -![ ](./train-on-cloud/static/image/workflow/3_workflow_scenario.png) +![ ](./static/image/3_workflow_scenario.png) \ No newline at end of file diff --git a/docs/train/train-on-cloud/static/image/workflow/1_workflow.png b/docs/workflow/compound-ai-workflows/static/image/1_workflow.png similarity index 100% rename from docs/train/train-on-cloud/static/image/workflow/1_workflow.png rename to docs/workflow/compound-ai-workflows/static/image/1_workflow.png diff --git a/docs/train/train-on-cloud/static/image/workflow/2_journey.png b/docs/workflow/compound-ai-workflows/static/image/2_journey.png similarity index 100% rename from docs/train/train-on-cloud/static/image/workflow/2_journey.png rename to docs/workflow/compound-ai-workflows/static/image/2_journey.png diff --git a/docs/train/train-on-cloud/static/image/workflow/3_workflow_scenario.png b/docs/workflow/compound-ai-workflows/static/image/3_workflow_scenario.png similarity index 100% rename from docs/train/train-on-cloud/static/image/workflow/3_workflow_scenario.png rename to docs/workflow/compound-ai-workflows/static/image/3_workflow_scenario.png diff --git a/docs/workflow/index.md b/docs/workflow/index.md new file mode 100644 index 0000000..5e92266 --- /dev/null +++ b/docs/workflow/index.md @@ -0,0 +1,16 @@ +--- +sidebar_position: 1 +--- + +# What is TensorOpera Workflow? + +As the AI space continuously evolves, so does the technical complexity of supporting emerging AI tools. New requirements are introduced requiring complex system designs and data flows that allow to combine ad-hoc data and model pipelines into cohesive workflows. Some typical AI workflows include: + +1. Model deployment workflows starting from data gathering and pre-processing to model training and evaluation. +2. Compound AI workflows tying together pipelines from different data and models. + +This is where the TensorOpera Workflow API becomes essential. The Workflow API offers an intuitive general-purpose interface to outline system components, tasks and their interdependencies. + + +![ ](./static/image/tensoropera_compoundai_system.png) + diff --git a/docs/workflow/static/image/tensoropera_compoundai_system.png b/docs/workflow/static/image/tensoropera_compoundai_system.png new file mode 100644 index 0000000..287a9ea Binary files /dev/null and b/docs/workflow/static/image/tensoropera_compoundai_system.png differ diff --git a/docusaurus.config.js b/docusaurus.config.js index b1aa3a8..54a00a0 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -150,16 +150,22 @@ const config = { label: 'Train', }, { - type: 'docSidebar', - sidebarId: 'federate', - position: 'left', - label: 'Federate', + type: "docSidebar", + sidebarId: "federate", + position: "left", + label: "Federate", }, { - type: 'docSidebar', - sidebarId: 'storage', - position: 'left', - label: 'Storage', + type: "docSidebar", + sidebarId: "workflow", + position: "left", + label: "Workflow", + }, + { + type: "docSidebar", + sidebarId: "storage", + position: "left", + label: "Storage", }, { type: "docSidebar", diff --git a/sidebars-default.js b/sidebars-default.js index 35fa00b..0b27de4 100644 --- a/sidebars-default.js +++ b/sidebars-default.js @@ -20,6 +20,7 @@ const sidebars = { train: [{ type: 'autogenerated', dirName: 'train' }], deploy: [{ type: 'autogenerated', dirName: 'deploy' }], federate: [{ type: 'autogenerated', dirName: 'federate' }], + workflow: [{ type: 'autogenerated', dirName: 'workflow' }], storage: [{ type: 'autogenerated', dirName: 'storage' }], community: [{ type: 'autogenerated', dirName: 'community' }], shareAndEarn: [{ type: 'autogenerated', dirName: 'share-and-earn'}]