From a8c2cee43a7545ff9106dc88495ecf5a29b19971 Mon Sep 17 00:00:00 2001 From: saradufour Date: Wed, 16 Apr 2025 16:09:01 +0200 Subject: [PATCH 1/4] Finished the first part of the assignment - Filling in the Dockerfile.train - adjusitng the train.py with the correct workingdir not to have bugs. - adjusting the README.md to view the progress of the assignment. --- Dockerfile.train | 17 +++++++++++++++-- README.md | 18 +++++++++--------- iris_model.pkl | Bin 0 -> 5859 bytes train.py | 2 +- 4 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 iris_model.pkl diff --git a/Dockerfile.train b/Dockerfile.train index 06ffaf9..0b7cec3 100644 --- a/Dockerfile.train +++ b/Dockerfile.train @@ -1,12 +1,25 @@ -FROM +FROM python:3.9-slim # TODO: Set a working directory +WORKDIR /my-workdir # TODO: Copy the requirements.txt file to the working directory +COPY requirements.txt /my-workdir/ + # TODO: Install the Python dependencies +RUN pip install --no-cache-dir -r requirements.txt # TODO: Copy the training script (train.py) to the working directory +COPY train.py /my-workdir/ # TODO: Run the training script that generates the model -CMD [...] +CMD ["python", "train.py"] + + +# command for image build +# docker build . --tag python-train-sd:v1.0.0 -f Dockerfile.train + +# command to run the docker image +# docker run -v .:/my-workdir python-train-sd:v1.0.0 +# output ot the docker run: Model training complete and saved as iris_model.pkl \ No newline at end of file diff --git a/README.md b/README.md index 8098afb..7221d57 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ In this project, you will train, run and serve a machine learning model using Do ## Deliverables -- [ ] Clone this repository to your personal github account -- [ ] Containerize training the machine learning model +- [X] Clone this repository to your personal github account +- [X] Containerize training the machine learning model - [ ] Containerize serving of the machine learning model -- [ ] Train and run the machine learning model using Docker +- [X] Train and run the machine learning model using Docker - [ ] Run the Docker container serving the machine learning model - [ ] Store the Docker images on your personal account on Docker Hub - [ ] Provide the resulting Dockerfiles in GitHub @@ -20,12 +20,12 @@ In this project, you will train, run and serve a machine learning model using Do Complete file named `Dockerfile.train` -- Copy requirements.txt and install dependencies -- Copy train.py to the working directory -- Set the command to run train.py -- Run the training of the model on your computer -- Document the command as comment in the Dockerfile -- Store the created Dockerfile in your cloned github repository +[X] Copy requirements.txt and install dependencies +[X] Copy train.py to the working directory +[X] Set the command to run train.py +[X] Run the training of the model on your computer +[X] Document the command as comment in the Dockerfile +[X] Store the created Dockerfile in your cloned github repository ## Proposed steps - containerize and serve the machine learning model diff --git a/iris_model.pkl b/iris_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..86ebedeb97a6762fb7d4b31799b7f886eec93d27 GIT binary patch literal 5859 zcmeHL+iw(A7~gK&?RHyA?^nc2Y*-Z5a#7KcY`LTbSE>PlL^PAx?X)v=cXpkbZ7oJE zF<_CS2{LOG{s6;^!RP}hctcA9%ofKBb$_yT3WM4A*M)5PN|AyYWpCN&snvzQE5cc235=MV->@S zk<6M3NQN`=^O9yMhW*$KG*)fMdVe+}nU-uRTvRf1vf+r<03V8MH2$i+-Hrn3CF||JzCLF&&YwiU zvXd+#r_;dL1B#zGQSZj(b)ER{8s zSSFvgG?5q<;uS6PG^#5$g-7mtrn5J5GHxG)95BdeE|)c|A~M#`A|t4+^>N|Ln9E4X zOg)6u2xtMc0@?t}0V@Ej0c!wj0S^Hl2CM@-0(ca_lhzH`2-pnh0c-(m1@QDe4%i0h z13UrP0eA|q6R-=g8}Kw>58zoq62Lj%3*a1buAc+&T#W!m0WSbv0=x{^4|oM|0B{g+ z2=E%=H&#nj4l5}uYna89Cdv*{g{^QYtD(vj{a0V|*4#bd zw3_ZUPP-Zx{rUK@cX(^U`6_o^%%6=5y++iJxa%UwAJI;Ot1slQq206i+%$Fz`Ea+e z`|H@WZSJ~=qaNd#!u1Z+qxc%!bs^u5 zenZ%M`g8dWpnHKN>AtRIg*;iovi!cH05sm7ju7l@y7^xKI0q&jc% z`SvRK0JOzy!0GXJ&=CAFF)G>{4y(W~qhV|Qw z@sfRt*ZFw>Q~vrOzNX#!5Oq%VPx@0>pLD*4Fdo`Z0ORrE6L!4kL*%s;hDgpU*qyc{E6p`&L=wG{QGNl>qq2`_UoOmRG-3bF@Iv7;`8<| z_TxW)o6sNS*IO6ZFtE1H3Itst0Kf1S62FufHB z1A)LVqn*7+wp_sd(K$95*Je(~PTuuF(LCynEv%oM@XLSt(S_}2YA+Ve`{fokF81^9 z^!;}H=w&ExcWqRQQf!m?tkLc%opFE{%PLz8~JbCmpuAj^qV-!1wQ-jOk2mh zh4^>x*ZkT1_t)kfA-lUed;N%|5s0WzxDQKohONEa-bwO`0+%6U#NJ4`yc6gqXEQi6Z$7KbY48s=c0G)usX86M!i#T5rB<{gZyv6TLtO`D3+)DoV zMU*!%ywJnB%A;dO7`c5uBM)Pu_diGe=ZnVwy}x}vbH=@M`u6$k@G)`yI`r4&C8&PsaWM@bf+V literal 0 HcmV?d00001 diff --git a/train.py b/train.py index 3128886..c920deb 100644 --- a/train.py +++ b/train.py @@ -12,6 +12,6 @@ model = clf.fit(iris.data, iris.target_names[iris.target]) #Save the trained model to the shared volume (make sure to use the correct path) -joblib.dump(model, '/app/models/iris_model.pkl') +joblib.dump(model, '/my-workdir/iris_model.pkl') print("Model training complete and saved as iris_model.pkl") From c5d32a75110d71a27fd9d8dd71ee26c8ca1ec383 Mon Sep 17 00:00:00 2001 From: saradufour Date: Thu, 24 Apr 2025 13:58:30 +0200 Subject: [PATCH 2/4] Adding part 2 of the docker project to github. (infer/server), generating a dockerhub account and storing the images. --- Dockerfile.infer | 39 +++++++++++++++++++++++++++++++++++---- README.md | 18 +++++++++--------- server.py | 2 +- train copy.py | 17 +++++++++++++++++ 4 files changed, 62 insertions(+), 14 deletions(-) create mode 100644 train copy.py diff --git a/Dockerfile.infer b/Dockerfile.infer index 9c9b020..3c8d50a 100644 --- a/Dockerfile.infer +++ b/Dockerfile.infer @@ -1,7 +1,38 @@ -WORKDIR /app -COPY requirements.txt . -COPY server.py . +#WORKDIR /app +#COPY requirements.txt . +#COPY server.py . +#FROM python:3.9-slim +#CMD ["python", "server.py"] +#RUN pip install --no-cache-dir -r requirements.txt +#EXPOSE 8080 + + FROM python:3.9-slim -CMD ["python", "server.py"] + +# TODO: Set a working directory +WORKDIR /my-workdir + +# TODO: Copy the requirements.txt file to the working directory +COPY requirements.txt /my-workdir/ + +# TODO: Install the Python dependencies RUN pip install --no-cache-dir -r requirements.txt + +# TODO: Copy the server script (server.py.py) to the working directory +COPY server.py /my-workdir/ + +#TODO: document which port(s) the container is intended to listen on at runtime EXPOSE 8080 + +# TODO: Run the server script that generates the model +CMD ["python", "server.py"] + + +# command for image build +#docker build . --tag python-server-sd:v1.0.0 -f Dockerfile.infer + +# command to run the docker image +#docker run --detach --name python_server_docker -p 4441:8080 -v .:/my-workdir python-server-sd:v1.0.0 + +#access the running docker locally via: http://127.0.0.1:4441/ + diff --git a/README.md b/README.md index 7221d57..49cd222 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,11 @@ In this project, you will train, run and serve a machine learning model using Do - [X] Clone this repository to your personal github account - [X] Containerize training the machine learning model -- [ ] Containerize serving of the machine learning model +- [X] Containerize serving of the machine learning model - [X] Train and run the machine learning model using Docker -- [ ] Run the Docker container serving the machine learning model -- [ ] Store the Docker images on your personal account on Docker Hub -- [ ] Provide the resulting Dockerfiles in GitHub +- [X] Run the Docker container serving the machine learning model +- [X] Store the Docker images on your personal account on Docker Hub +- [X] Provide the resulting Dockerfiles in GitHub - [ ] Build an Apptainer image on a HPC of your choice - [ ] Provide the logs of the slurm job in GitHub - [ ] Document the steps in a text document in GitHub @@ -29,14 +29,14 @@ Complete file named `Dockerfile.train` ## Proposed steps - containerize and serve the machine learning model -- Correct the order of the instructions in the Dockerfile.infer -- Document the steps in the Dockerfile.infer as comments -- Document the succesful `docker run` command in the Dockerfile.infer as a comment +[X] Correct the order of the instructions in the Dockerfile.infer +[X] Document the steps in the Dockerfile.infer as comments +[X] Document the succesful `docker run` command in the Dockerfile.infer as a comment ## Proposed steps - store images on Dockerhub and build an Apptainer image on the HPC -- Create an account on Dockerhub -- Store the built images on your account +[X] Create an account on Dockerhub +[X] Store the built images on your account - Create a shell script on the HPC of your preference - Store the shell script in your cloned github repository diff --git a/server.py b/server.py index c33e788..bc0913d 100644 --- a/server.py +++ b/server.py @@ -7,7 +7,7 @@ app = Flask(__name__) # Check if the model file exists and wait until it does -model_path = '/app/models/iris_model.pkl' +model_path = '/my-workdir/iris_model.pkl' while not os.path.exists(model_path): print(f"Waiting for model file at {model_path}...") diff --git a/train copy.py b/train copy.py new file mode 100644 index 0000000..c920deb --- /dev/null +++ b/train copy.py @@ -0,0 +1,17 @@ +from sklearn.svm import SVC +from sklearn import datasets +import joblib + +#Load the Iris dataset +iris = datasets.load_iris() + +#Create an SVM classifier +clf = SVC() + +#Train the model using the iris dataset +model = clf.fit(iris.data, iris.target_names[iris.target]) + +#Save the trained model to the shared volume (make sure to use the correct path) +joblib.dump(model, '/my-workdir/iris_model.pkl') + +print("Model training complete and saved as iris_model.pkl") From 700d11207e18b92a9949738b850edf37b72a210e Mon Sep 17 00:00:00 2001 From: saradufour Date: Thu, 24 Apr 2025 14:57:52 +0200 Subject: [PATCH 3/4] adding apptainer part, finalizing the documentation --- README.md | 32 +++++++++++------------ Step-By-step.md | 49 ++++++++++++++++++++++++++++++++++++ VSC/docker-project.sh | 17 +++++++++++++ VSC/python-server-sd-OUT.log | 21 ++++++++++++++++ VSC/python-train-sd-OUT.log | 21 ++++++++++++++++ VSC/slurm-20167414.out | 2 ++ 6 files changed, 126 insertions(+), 16 deletions(-) create mode 100644 Step-By-step.md create mode 100644 VSC/docker-project.sh create mode 100644 VSC/python-server-sd-OUT.log create mode 100644 VSC/python-train-sd-OUT.log create mode 100644 VSC/slurm-20167414.out diff --git a/README.md b/README.md index 49cd222..89319c9 100644 --- a/README.md +++ b/README.md @@ -12,33 +12,33 @@ In this project, you will train, run and serve a machine learning model using Do - [X] Run the Docker container serving the machine learning model - [X] Store the Docker images on your personal account on Docker Hub - [X] Provide the resulting Dockerfiles in GitHub -- [ ] Build an Apptainer image on a HPC of your choice -- [ ] Provide the logs of the slurm job in GitHub -- [ ] Document the steps in a text document in GitHub +- [X] Build an Apptainer image on a HPC of your choice +- [X] Provide the logs of the slurm job in GitHub +- [X] Document the steps in a text document in GitHub (Step-By-Step.md) ## Proposed steps - containerize and run training the machine learning model Complete file named `Dockerfile.train` -[X] Copy requirements.txt and install dependencies -[X] Copy train.py to the working directory -[X] Set the command to run train.py -[X] Run the training of the model on your computer -[X] Document the command as comment in the Dockerfile -[X] Store the created Dockerfile in your cloned github repository +- [X] Copy requirements.txt and install dependencies +- [X] Copy train.py to the working directory +- [X] Set the command to run train.py +- [X] Run the training of the model on your computer +- [X] Document the command as comment in the Dockerfile +- [X] Store the created Dockerfile in your cloned github repository ## Proposed steps - containerize and serve the machine learning model -[X] Correct the order of the instructions in the Dockerfile.infer -[X] Document the steps in the Dockerfile.infer as comments -[X] Document the succesful `docker run` command in the Dockerfile.infer as a comment +- [X] Correct the order of the instructions in the Dockerfile.infer +- [X] Document the steps in the Dockerfile.infer as comments +- [X] Document the succesful `docker run` command in the Dockerfile.infer as a comment ## Proposed steps - store images on Dockerhub and build an Apptainer image on the HPC -[X] Create an account on Dockerhub -[X] Store the built images on your account -- Create a shell script on the HPC of your preference -- Store the shell script in your cloned github repository +- [X] Create an account on Dockerhub +- [X] Store the built images on your account +- [X]Create a shell script on the HPC of your preference +- [X]Store the shell script in your cloned github repository diff --git a/Step-By-step.md b/Step-By-step.md new file mode 100644 index 0000000..ce8551e --- /dev/null +++ b/Step-By-step.md @@ -0,0 +1,49 @@ +# Clone this repository to your personal github account +git clone git@github.com:saradufour/project_docker_microcredential.git + +# Containerize training the machine learning model + +command for image build +`docker build . --tag python-train-sd:v1.0.0 -f Dockerfile.train` + + +# Containerize serving of the machine learning model + +command for image build +`docker build . --tag python-server-sd:v1.0.0 -f Dockerfile.infer` + + +# Train and run the machine learning model using Docker + + +command to run the docker image (train) +`docker run -v .:/my-workdir python-train-sd:v1.0.0` +output ot the docker run: Model training complete and saved as iris_model.pkl + + +# Run the Docker container serving the machine learning model + +command to run the docker image (server) +`docker run --detach --name python_server_docker -p 4441:8080 -v .:/my-workdir python-server-sd:v1.0.0` + +access the running docker locally via: http://127.0.0.1:4441/ + + +# Store the Docker images on your personal account on Docker Hub + +`docker images` +`docker login` +`docker tag python-server-sd:v1.0.0 saradufour/python-server-sd:v1.0.0` +`docker push saradufour/python-server-sd:v1.0.0` +`docker tag python-train-sd:v1.0.0 saradufour/python-train-sd:v1.0.0` +`docker push saradufour/python-train-sd:v1.0.0` + + +# Provide the resulting Dockerfiles in GitHub +`git add *` +`git commit -m "..."` + +# Apptainer +see VSC folder in github repository + +`sbatch docker-project.sh` diff --git a/VSC/docker-project.sh b/VSC/docker-project.sh new file mode 100644 index 0000000..6d06692 --- /dev/null +++ b/VSC/docker-project.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +#SBATCH --job-name=jbuild-apptainer +#SBATCH --partition=donphan +#SBATCH --mem=8G +#SBATCH --time=00:30:00 + + +LOG_DIR="$VSC_SCRATCH/logs" +mkdir -p $LOG_DIR + +echo "build containers on VSC" + +apptainer build --fakeroot python-train-sd.sif docker://saradufour/python-train-sd:v1.0.0 > $LOG_DIR/python-train-sd-OUT.log 2>&1 +apptainer build --fakeroot python-server-sd.sif docker://saradufour/python-server-sd:v1.0.0 > $LOG_DIR/python-server-sd-OUT.log 2>&1 + +echo "You can now run your container" diff --git a/VSC/python-server-sd-OUT.log b/VSC/python-server-sd-OUT.log new file mode 100644 index 0000000..99b3052 --- /dev/null +++ b/VSC/python-server-sd-OUT.log @@ -0,0 +1,21 @@ +INFO: Starting build... +Copying blob sha256:24c9ec169934c78008ef30494c5735cce146c038004a2cf1bd5f63b674c75093 +Copying blob sha256:8a628cdd7ccc83e90e5a95888fcb0ec24b991141176c515ad101f12d6433eb96 +Copying blob sha256:74018f7cfa8f2965fd86b13c38f71417bc846e071a5f5bb5ae569ccb5a6e7248 +Copying blob sha256:a0b0cfc480ce03c723a597904bcfbf28c71438c689e6d5097c2332835f67a40c +Copying blob sha256:97d21b95fb00ac3b08975ab6f8709f3a7e35a05d75e2f9a70fa95348279dac27 +Copying blob sha256:d4802e1316679b361e2ebd439a0153a86f1a74de1f6490bce82b6985cdd27bd5 +Copying blob sha256:bc05d8da204fd477b8bf4f4aeef2a6ea6694053102449fc04e709c6203026eb8 +Copying blob sha256:47432841852a4fdb7690e090a9a51676251d42705f1af2e5a1eba975fbeb092b +Copying config sha256:472c6673fd87607215351b99a6779289b86b2a364d96f2a662cffbd859f146e2 +Writing manifest to image destination +2025/04/24 14:41:05 info unpack layer: sha256:8a628cdd7ccc83e90e5a95888fcb0ec24b991141176c515ad101f12d6433eb96 +2025/04/24 14:41:07 info unpack layer: sha256:74018f7cfa8f2965fd86b13c38f71417bc846e071a5f5bb5ae569ccb5a6e7248 +2025/04/24 14:41:07 info unpack layer: sha256:a0b0cfc480ce03c723a597904bcfbf28c71438c689e6d5097c2332835f67a40c +2025/04/24 14:41:07 info unpack layer: sha256:97d21b95fb00ac3b08975ab6f8709f3a7e35a05d75e2f9a70fa95348279dac27 +2025/04/24 14:41:07 info unpack layer: sha256:d4802e1316679b361e2ebd439a0153a86f1a74de1f6490bce82b6985cdd27bd5 +2025/04/24 14:41:07 info unpack layer: sha256:24c9ec169934c78008ef30494c5735cce146c038004a2cf1bd5f63b674c75093 +2025/04/24 14:41:07 info unpack layer: sha256:bc05d8da204fd477b8bf4f4aeef2a6ea6694053102449fc04e709c6203026eb8 +2025/04/24 14:41:11 info unpack layer: sha256:47432841852a4fdb7690e090a9a51676251d42705f1af2e5a1eba975fbeb092b +INFO: Creating SIF file... +INFO: Build complete: python-server-sd.sif diff --git a/VSC/python-train-sd-OUT.log b/VSC/python-train-sd-OUT.log new file mode 100644 index 0000000..424ec28 --- /dev/null +++ b/VSC/python-train-sd-OUT.log @@ -0,0 +1,21 @@ +INFO: Starting build... +Copying blob sha256:24c9ec169934c78008ef30494c5735cce146c038004a2cf1bd5f63b674c75093 +Copying blob sha256:8a628cdd7ccc83e90e5a95888fcb0ec24b991141176c515ad101f12d6433eb96 +Copying blob sha256:74018f7cfa8f2965fd86b13c38f71417bc846e071a5f5bb5ae569ccb5a6e7248 +Copying blob sha256:a0b0cfc480ce03c723a597904bcfbf28c71438c689e6d5097c2332835f67a40c +Copying blob sha256:97d21b95fb00ac3b08975ab6f8709f3a7e35a05d75e2f9a70fa95348279dac27 +Copying blob sha256:d4802e1316679b361e2ebd439a0153a86f1a74de1f6490bce82b6985cdd27bd5 +Copying blob sha256:bc05d8da204fd477b8bf4f4aeef2a6ea6694053102449fc04e709c6203026eb8 +Copying blob sha256:cf612e53a4c896366012dc36f6caea05fdd787b1ef4f52405a61d081f0fc05a0 +Copying config sha256:0ac2dd017a895bf7cb7bda4e539f8b1cd9ad226083ac43541704cad96e34dab9 +Writing manifest to image destination +2025/04/24 14:40:06 info unpack layer: sha256:8a628cdd7ccc83e90e5a95888fcb0ec24b991141176c515ad101f12d6433eb96 +2025/04/24 14:40:07 info unpack layer: sha256:74018f7cfa8f2965fd86b13c38f71417bc846e071a5f5bb5ae569ccb5a6e7248 +2025/04/24 14:40:07 info unpack layer: sha256:a0b0cfc480ce03c723a597904bcfbf28c71438c689e6d5097c2332835f67a40c +2025/04/24 14:40:08 info unpack layer: sha256:97d21b95fb00ac3b08975ab6f8709f3a7e35a05d75e2f9a70fa95348279dac27 +2025/04/24 14:40:08 info unpack layer: sha256:d4802e1316679b361e2ebd439a0153a86f1a74de1f6490bce82b6985cdd27bd5 +2025/04/24 14:40:08 info unpack layer: sha256:24c9ec169934c78008ef30494c5735cce146c038004a2cf1bd5f63b674c75093 +2025/04/24 14:40:08 info unpack layer: sha256:bc05d8da204fd477b8bf4f4aeef2a6ea6694053102449fc04e709c6203026eb8 +2025/04/24 14:40:11 info unpack layer: sha256:cf612e53a4c896366012dc36f6caea05fdd787b1ef4f52405a61d081f0fc05a0 +INFO: Creating SIF file... +INFO: Build complete: python-train-sd.sif diff --git a/VSC/slurm-20167414.out b/VSC/slurm-20167414.out new file mode 100644 index 0000000..2185e43 --- /dev/null +++ b/VSC/slurm-20167414.out @@ -0,0 +1,2 @@ +build containers on VSC +You can now run your container From 547f11523ba30f7cc84f2ee1de4de4e0f4119ae4 Mon Sep 17 00:00:00 2001 From: saradufour <43069189+saradufour@users.noreply.github.com> Date: Thu, 24 Apr 2025 14:59:00 +0200 Subject: [PATCH 4/4] finishing touches to the readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 89319c9..951d5bd 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ Complete file named `Dockerfile.train` - [X] Create an account on Dockerhub - [X] Store the built images on your account -- [X]Create a shell script on the HPC of your preference -- [X]Store the shell script in your cloned github repository +- [X] Create a shell script on the HPC of your preference +- [X] Store the shell script in your cloned github repository