feat(python): support multiple versions

.github/workflows/release-older-version-sdks.yml

@@ -6,12 +6,16 @@ env:
   MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD_2024OCT }}
   MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
   JAVA_SDK_LOCATION_v1: "java/v1" 
-  JAVA_SDK_LOCATION_v2: "java/v2" 
+  JAVA_SDK_LOCATION_v2: "java/v2"
+  PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
+  PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+  PYTHON_SDK_LOCATION_v2: "python/fds.protobuf.stach.v2"
+
 on:
   workflow_dispatch:
     inputs:
       SDKs:
-        description: 'Which SDK to release? Options are "java-v1" and "java-v2"'
+        description: 'Which SDK to release? Options are "java-v1", "java-v2" and "python-v2"'
         required: true
         default: all
 
@@ -68,4 +72,31 @@ jobs:
     - name: Publish to Apache Maven Central
       run: |
         cd ${{ env.JAVA_SDK_LOCATION_v2 }}
-        mvn -Psign-artifacts verify deploy
+        mvn -Psign-artifacts verify deploy
+
+  python-v2:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch' 
+      && github.event.inputs.SDKs == 'python-v2'
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+
+    - name: Build and publish to PyPI
+      env:
+        TWINE_USERNAME: ${{ env.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ env.PYPI_PASSWORD }}
+      run: |
+        cd ${{ env.PYTHON_SDK_LOCATION_v2 }}
+        python setup.py sdist bdist_wheel
+        twine upload dist/* --verbose

.gitignore

@@ -10,6 +10,10 @@ node_modules/
 js/node_modules/
 python/fds.protobuf.stach/dist/*.*
 python/fds.protobuf.stach/fds.protobuf.stach.egg-info
+python/fds.protobuf.stach.v2/dist/*.*
+python/fds.protobuf.stach.v2/fds.protobuf.stach.v2.egg-info
+python/fds.protobuf.stach.v3/dist/*.*
+python/fds.protobuf.stach.v3/fds.protobuf.stach.v3.egg-info
 r/*.tar.gz
 r/.RData
 r/.Rhistory
@@ -21,4 +25,4 @@ js/.yarn/*
 !js/.yarn/versions
 .DS_Store
 go/pkg/
-artifacts
+artifacts

docker-compose.generate.yml

@@ -19,7 +19,9 @@ services:
       context: .
       dockerfile: python/Dockerfile.generate
     volumes:
-    - ${SCHEMA}/v3:/schema
+    - ${SCHEMA}/v1:/schema/v1
+    - ${SCHEMA}/v2:/schema/v2
+    - ${SCHEMA}/v3:/schema/v3
     - ./python:/python
   generate_java:
     build:
@@ -37,10 +39,10 @@ services:
     volumes:
     - ${SCHEMA}/v3:/schema
     - ./r:/r
-  generate_go:
-    build:
-      context: .
-      dockerfile: go/Dockerfile.generate
-    volumes:
-    - ${SCHEMA}/v3:/schema
-    - ./go:/golang
+  # generate_go:
+  #   build:
+  #     context: .
+  #     dockerfile: go/Dockerfile.generate
+  #   volumes:
+  #   - ${SCHEMA}/v3:/schema
+  #   - ./go:/golang

dotnet/Dockerfile.build

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/dotnet/core/sdk:2.2
+FROM mcr.microsoft.com/dotnet/sdk:6.0
 
 LABEL org.label-schema.license="Apache-2.0" \
       org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \
@@ -9,4 +9,4 @@ COPY dotnet /src
 
 RUN /src/CI/build-sdk.sh
 
-ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"]
+ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"]

dotnet/Dockerfile.generate

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/dotnet/core/sdk:2.2
+FROM mcr.microsoft.com/dotnet/sdk:6.0
 
 LABEL org.label-schema.license="Apache-2.0" \
       org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \

go/CI/generate-sdk.sh

@@ -11,7 +11,7 @@ rm -f $BASE_PATH/$PACKAGE_PATH/**/*.pb.go
 
 echo Removed old generated code
 
-PROTOFILES=$(find $SCHEMA_PATH/$PACKAGE_PATH -iname "*.proto")
+PROTOFILES=$(find $SCHEMA_PATH -iname "*.proto")
 protoc --proto_path $SCHEMA_PATH --go_out paths=source_relative:$BASE_PATH $PROTOFILES
 
 cd $BASE_PATH/$PACKAGE_PATH

go/Dockerfile.build

@@ -1,4 +1,4 @@
-FROM golang:1.14.6
+FROM golang:1.23
 
 LABEL org.label-schema.license="Apache-2.0" \
       org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \
@@ -7,4 +7,4 @@ LABEL org.label-schema.license="Apache-2.0" \
 
 COPY go /src
 
-RUN /src/CI/build-sdk.sh
+RUN /src/CI/build-sdk.sh

go/Dockerfile.generate

@@ -1,19 +1,20 @@
-FROM golang:1.14.6
+FROM golang:1.23-alpine
 
 LABEL org.label-schema.license="Apache-2.0" \
       org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \
       org.label-schema.description="To generate Go Stach package" \
       maintainer="[email protected]"
 
-RUN apt-get update && \
-    apt-get -y install unzip
+RUN apk update && apk add curl unzip bash git
 
 ARG PROTOC_ZIP=protoc-3.19.4-linux-x86_64.zip
 RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/$PROTOC_ZIP
 RUN unzip -o $PROTOC_ZIP -d /usr/local bin/protoc
 RUN unzip -o $PROTOC_ZIP -d /usr/local 'include/*'
 RUN rm -f $PROTOC_ZIP
 
-RUN go get google.golang.org/protobuf/cmd/protoc-gen-go
+RUN chmod +x /usr/local/bin/protoc
+
+RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
 
 ENTRYPOINT ["bash", "/golang/CI/generate-sdk.sh"]

java/CI/generate-sdk.sh

@@ -8,8 +8,18 @@ BASE_PATH_v1=/java/v1
 BASE_PATH_v2=/java/v2
 BASE_PATH_v3=/java/v3
 
-rm -f $BASE_PATH/**/*.java
-
+# Backup section - store both file and its directory
+for BASE in $BASE_PATH_v1 $BASE_PATH_v2 $BASE_PATH_v3; do
+    NULLVALUES_FILE=$(find "$BASE/$PACKAGE_PATH" -name "NullValues.java" -type f | head -1)
+    if [ -n "$NULLVALUES_FILE" ]; then
+        cp "$NULLVALUES_FILE" "/tmp/NullValues_$(basename $BASE).java"
+        # Store the original directory path
+        echo "$(dirname "$NULLVALUES_FILE")" > "/tmp/NullValues_$(basename $BASE).dir"
+        echo "Backed up: $NULLVALUES_FILE"
+    fi
+done
+
+rm -f $BASE_PATH/$PACKAGE_PATH/**/*Proto.java
 
 mkdir --parent $BASE_PATH_v1/$PACKAGE_PATH
 mkdir --parent $BASE_PATH_v2/$PACKAGE_PATH
@@ -20,18 +30,25 @@ echo Removed old generated code
 # cd $BASE_PATH
 # mvn clean install -q -P generate-stach-schema -f pom.xml
 
-cd $BASE_PATH_v1
-mvn clean install -q -P generate-stach-schema -f pom.xml
+mvn -f $BASE_PATH_v1/pom.xml clean install -q -P generate-stach-schema
 echo Generated java stach v1 files
 
-cd ../..
-cd $BASE_PATH_v2
-mvn clean install -q -P generate-stach-schema -f pom.xml
+mvn -f $BASE_PATH_v2/pom.xml clean install -q -P generate-stach-schema
 echo Generated java stach v2 files
 
-cd ../..
-cd $BASE_PATH_v3
-mvn clean install -q -P generate-stach-schema -f pom.xml
+mvn -f $BASE_PATH_v3/pom.xml clean install -q -P generate-stach-schema
 echo Generated java stach v3 files
 
-echo Produced new generated code
+echo Produced new generated code
+
+# Restore section - use stored directory path
+for BASE in $BASE_PATH_v1 $BASE_PATH_v2 $BASE_PATH_v3; do
+    if [ -f "/tmp/NullValues_$(basename $BASE).java" ]; then
+        ORIGINAL_DIR=$(cat "/tmp/NullValues_$(basename $BASE).dir")
+        mkdir -p "$ORIGINAL_DIR"  # Ensure directory exists
+        cp "/tmp/NullValues_$(basename $BASE).java" "$ORIGINAL_DIR/NullValues.java"
+        rm "/tmp/NullValues_$(basename $BASE).java"
+        rm "/tmp/NullValues_$(basename $BASE).dir"
+        echo "Restored: $ORIGINAL_DIR/NullValues.java"
+    fi
+done

java/Dockerfile.build

@@ -1,4 +1,4 @@
-FROM maven:3.6.3-jdk-8
+FROM maven:3.9.6-eclipse-temurin-17
 
 LABEL org.label-schema.license="Apache-2.0" \
       org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \
@@ -9,4 +9,4 @@ COPY java /src
 
 RUN /src/CI/build-sdk.sh
 
-ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"]
+ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"]

java/Dockerfile.generate

@@ -1,8 +1,8 @@
-FROM maven:3.6.3-jdk-8
+FROM maven:3.9.6-eclipse-temurin-17
 
 LABEL org.label-schema.license="Apache-2.0" \
       org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \
       org.label-schema.description="To generate Java Stach package" \
       maintainer="[email protected]"
 
-ENTRYPOINT ["bash", "/java/CI/generate-sdk.sh"]
+ENTRYPOINT ["bash", "/java/CI/generate-sdk.sh"]

java/v1/pom.xml

@@ -166,4 +166,4 @@
             <version>${version.protobuf}</version>
         </dependency>
     </dependencies>
-</project>
+</project>

python/CI/generate-sdk.sh

@@ -2,19 +2,30 @@
 set -e
 
 SCHEMA_PATH=/schema # Location of Protobuf schema
-BASE_PATH=/python/fds.protobuf.stach.v3 # Base location of Python generated classes
-PACKAGE_PATH=fds/protobuf/stach/v3
+BASE_PATH_V3=/python/fds.protobuf.stach.v3 # Base location of Python generated classes
+BASE_PATH_V2=/python/fds.protobuf.stach.v2 # Base location of Python generated classes
+PACKAGE_PATH_V3=v3/fds/protobuf/stach/v3
+PACKAGE_PATH_V2=v2/fds/protobuf/stach/v2
 
-rm -f $BASE_PATH/$PACKAGE_PATH/**/*_pb2.py
+# echo Removed old generated code
+rm -f $BASE_PATH_V2/$PACKAGE_PATH_V2/**/*_pb2.py
+rm -f $BASE_PATH_V3/$PACKAGE_PATH_V3/**/*_pb2.py
 
-echo Removed old generated code
+mkdir -p "$BASE_PATH_V3"
+mkdir -p "$BASE_PATH_V2"
 
-PROTOFILES=$(find $SCHEMA_PATH/$PACKAGE_PATH -iname "*.proto")
-protoc --proto_path $SCHEMA_PATH --python_out $BASE_PATH $PROTOFILES
+PROTOFILES=$(find $SCHEMA_PATH/$PACKAGE_PATH_V3 -iname "*.proto")
+protoc --proto_path $SCHEMA_PATH/v3 --python_out $BASE_PATH_V3 $PROTOFILES
 
-for f in $(find $BASE_PATH/fds/protobuf/stach/v3 -type d -maxdepth 10); do
-	cd "$f"
-	touch __init__.py
+for f in $(find $BASE_PATH_V3/fds/protobuf/stach/v3 -type d -maxdepth 10); do
+	touch "$f/__init__.py"
+done;
+
+PROTOFILES=$(find $SCHEMA_PATH/$PACKAGE_PATH_V2 -iname "*.proto")
+protoc --proto_path $SCHEMA_PATH/v2 --python_out $BASE_PATH_V2 $PROTOFILES
+
+for f in $(find $BASE_PATH_V2/fds/protobuf/stach/v2 -type d -maxdepth 10); do
+	touch "$f/__init__.py"
 done;
 
 echo Produced new generated code

python/Dockerfile.build

@@ -1,4 +1,4 @@
-FROM python:3.8
+FROM python:3.9-slim
 
 LABEL org.label-schema.license="Apache-2.0" \
       org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \
@@ -9,4 +9,4 @@ COPY python /src
 
 RUN /src/CI/build-sdk.sh
 
-ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"]
+ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"]

python/Dockerfile.generate

@@ -1,12 +1,13 @@
-FROM python:3.8
+FROM python:3.9-slim
 
 LABEL org.label-schema.license="Apache-2.0" \
       org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \
       org.label-schema.description="To generate Python Stach package" \
       maintainer="[email protected]"
 
 RUN apt-get update && \
-    apt-get -y install unzip
+    apt-get -y install unzip git curl && \
+    rm -rf /var/lib/apt/lists/*
 
 ARG PROTOC_ZIP=protoc-3.19.4-linux-x86_64.zip
 RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/$PROTOC_ZIP

python/fds.protobuf.stach.v2/setup.py

@@ -0,0 +1,12 @@
+from setuptools import setup, find_packages
+
+setup(name='fds.protobuf.stach.v2',
+      version='1.0.3',
+      description='Stach library in python',
+      url='',
+      author='analytics-reporting',
+      author_email='[email protected]',
+      license='Apache License 2.0',
+      install_requires=["protobuf>=3.19.4"],
+      packages=find_packages(),
+      zip_safe=False)

python/fds.protobuf.stach.v3/setup.py

@@ -1,12 +1,12 @@
 from setuptools import setup, find_packages
 
 setup(name='fds.protobuf.stach.v3',
-      version='1.0.0',
+      version='1.0.1',
       description='Stach library in python',
       url='',
       author='analytics-reporting',
       author_email='[email protected]',
       license='Apache License 2.0',
-      install_requires=["protobuf==3.19.4"],
+      install_requires=["protobuf>=3.19.4"],
       packages=find_packages(),
       zip_safe=False)