diff --git a/.github/workflows/release-older-version-sdks.yml b/.github/workflows/release-older-version-sdks.yml index 095ba5c..e05306d 100644 --- a/.github/workflows/release-older-version-sdks.yml +++ b/.github/workflows/release-older-version-sdks.yml @@ -6,12 +6,16 @@ env: MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD_2024OCT }} MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} JAVA_SDK_LOCATION_v1: "java/v1" - JAVA_SDK_LOCATION_v2: "java/v2" + JAVA_SDK_LOCATION_v2: "java/v2" + PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + PYTHON_SDK_LOCATION_v2: "python/fds.protobuf.stach.v2" + on: workflow_dispatch: inputs: SDKs: - description: 'Which SDK to release? Options are "java-v1" and "java-v2"' + description: 'Which SDK to release? Options are "java-v1", "java-v2" and "python-v2"' required: true default: all @@ -68,4 +72,31 @@ jobs: - name: Publish to Apache Maven Central run: | cd ${{ env.JAVA_SDK_LOCATION_v2 }} - mvn -Psign-artifacts verify deploy \ No newline at end of file + mvn -Psign-artifacts verify deploy + + python-v2: + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' + && github.event.inputs.SDKs == 'python-v2' + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + + - name: Build and publish to PyPI + env: + TWINE_USERNAME: ${{ env.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ env.PYPI_PASSWORD }} + run: | + cd ${{ env.PYTHON_SDK_LOCATION_v2 }} + python setup.py sdist bdist_wheel + twine upload dist/* --verbose diff --git a/.gitignore b/.gitignore index 3b54a65..75b284b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ node_modules/ js/node_modules/ python/fds.protobuf.stach/dist/*.* python/fds.protobuf.stach/fds.protobuf.stach.egg-info +python/fds.protobuf.stach.v2/dist/*.* +python/fds.protobuf.stach.v2/fds.protobuf.stach.v2.egg-info +python/fds.protobuf.stach.v3/dist/*.* +python/fds.protobuf.stach.v3/fds.protobuf.stach.v3.egg-info r/*.tar.gz r/.RData r/.Rhistory @@ -21,4 +25,4 @@ js/.yarn/* !js/.yarn/versions .DS_Store go/pkg/ -artifacts \ No newline at end of file +artifacts diff --git a/docker-compose.generate.yml b/docker-compose.generate.yml index c0dbdb2..d283224 100644 --- a/docker-compose.generate.yml +++ b/docker-compose.generate.yml @@ -19,7 +19,9 @@ services: context: . dockerfile: python/Dockerfile.generate volumes: - - ${SCHEMA}/v3:/schema + - ${SCHEMA}/v1:/schema/v1 + - ${SCHEMA}/v2:/schema/v2 + - ${SCHEMA}/v3:/schema/v3 - ./python:/python generate_java: build: @@ -37,10 +39,10 @@ services: volumes: - ${SCHEMA}/v3:/schema - ./r:/r - generate_go: - build: - context: . - dockerfile: go/Dockerfile.generate - volumes: - - ${SCHEMA}/v3:/schema - - ./go:/golang \ No newline at end of file + # generate_go: + # build: + # context: . + # dockerfile: go/Dockerfile.generate + # volumes: + # - ${SCHEMA}/v3:/schema + # - ./go:/golang diff --git a/dotnet/Dockerfile.build b/dotnet/Dockerfile.build index 2688b8f..6f626c5 100644 --- a/dotnet/Dockerfile.build +++ b/dotnet/Dockerfile.build @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/core/sdk:2.2 +FROM mcr.microsoft.com/dotnet/sdk:6.0 LABEL org.label-schema.license="Apache-2.0" \ org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \ @@ -9,4 +9,4 @@ COPY dotnet /src RUN /src/CI/build-sdk.sh -ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"] \ No newline at end of file +ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"] diff --git a/dotnet/Dockerfile.generate b/dotnet/Dockerfile.generate index f906412..937f94a 100644 --- a/dotnet/Dockerfile.generate +++ b/dotnet/Dockerfile.generate @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/core/sdk:2.2 +FROM mcr.microsoft.com/dotnet/sdk:6.0 LABEL org.label-schema.license="Apache-2.0" \ org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \ diff --git a/go/CI/generate-sdk.sh b/go/CI/generate-sdk.sh index 2a48496..da9f196 100755 --- a/go/CI/generate-sdk.sh +++ b/go/CI/generate-sdk.sh @@ -11,7 +11,7 @@ rm -f $BASE_PATH/$PACKAGE_PATH/**/*.pb.go echo Removed old generated code -PROTOFILES=$(find $SCHEMA_PATH/$PACKAGE_PATH -iname "*.proto") +PROTOFILES=$(find $SCHEMA_PATH -iname "*.proto") protoc --proto_path $SCHEMA_PATH --go_out paths=source_relative:$BASE_PATH $PROTOFILES cd $BASE_PATH/$PACKAGE_PATH diff --git a/go/Dockerfile.build b/go/Dockerfile.build index 21d6458..bc41bce 100644 --- a/go/Dockerfile.build +++ b/go/Dockerfile.build @@ -1,4 +1,4 @@ -FROM golang:1.14.6 +FROM golang:1.23 LABEL org.label-schema.license="Apache-2.0" \ org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \ @@ -7,4 +7,4 @@ LABEL org.label-schema.license="Apache-2.0" \ COPY go /src -RUN /src/CI/build-sdk.sh \ No newline at end of file +RUN /src/CI/build-sdk.sh diff --git a/go/Dockerfile.generate b/go/Dockerfile.generate index f7e9fff..25b8d8e 100644 --- a/go/Dockerfile.generate +++ b/go/Dockerfile.generate @@ -1,12 +1,11 @@ -FROM golang:1.14.6 +FROM golang:1.23-alpine LABEL org.label-schema.license="Apache-2.0" \ org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \ org.label-schema.description="To generate Go Stach package" \ maintainer="analytics.api.support@factset.com" -RUN apt-get update && \ - apt-get -y install unzip +RUN apk update && apk add curl unzip bash git ARG PROTOC_ZIP=protoc-3.19.4-linux-x86_64.zip RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/$PROTOC_ZIP @@ -14,6 +13,8 @@ RUN unzip -o $PROTOC_ZIP -d /usr/local bin/protoc RUN unzip -o $PROTOC_ZIP -d /usr/local 'include/*' RUN rm -f $PROTOC_ZIP -RUN go get google.golang.org/protobuf/cmd/protoc-gen-go +RUN chmod +x /usr/local/bin/protoc + +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest ENTRYPOINT ["bash", "/golang/CI/generate-sdk.sh"] diff --git a/java/CI/generate-sdk.sh b/java/CI/generate-sdk.sh index e850008..cc356d4 100755 --- a/java/CI/generate-sdk.sh +++ b/java/CI/generate-sdk.sh @@ -8,8 +8,18 @@ BASE_PATH_v1=/java/v1 BASE_PATH_v2=/java/v2 BASE_PATH_v3=/java/v3 -rm -f $BASE_PATH/**/*.java - +# Backup section - store both file and its directory +for BASE in $BASE_PATH_v1 $BASE_PATH_v2 $BASE_PATH_v3; do + NULLVALUES_FILE=$(find "$BASE/$PACKAGE_PATH" -name "NullValues.java" -type f | head -1) + if [ -n "$NULLVALUES_FILE" ]; then + cp "$NULLVALUES_FILE" "/tmp/NullValues_$(basename $BASE).java" + # Store the original directory path + echo "$(dirname "$NULLVALUES_FILE")" > "/tmp/NullValues_$(basename $BASE).dir" + echo "Backed up: $NULLVALUES_FILE" + fi +done + +rm -f $BASE_PATH/$PACKAGE_PATH/**/*Proto.java mkdir --parent $BASE_PATH_v1/$PACKAGE_PATH mkdir --parent $BASE_PATH_v2/$PACKAGE_PATH @@ -20,18 +30,25 @@ echo Removed old generated code # cd $BASE_PATH # mvn clean install -q -P generate-stach-schema -f pom.xml -cd $BASE_PATH_v1 -mvn clean install -q -P generate-stach-schema -f pom.xml +mvn -f $BASE_PATH_v1/pom.xml clean install -q -P generate-stach-schema echo Generated java stach v1 files -cd ../.. -cd $BASE_PATH_v2 -mvn clean install -q -P generate-stach-schema -f pom.xml +mvn -f $BASE_PATH_v2/pom.xml clean install -q -P generate-stach-schema echo Generated java stach v2 files -cd ../.. -cd $BASE_PATH_v3 -mvn clean install -q -P generate-stach-schema -f pom.xml +mvn -f $BASE_PATH_v3/pom.xml clean install -q -P generate-stach-schema echo Generated java stach v3 files -echo Produced new generated code \ No newline at end of file +echo Produced new generated code + +# Restore section - use stored directory path +for BASE in $BASE_PATH_v1 $BASE_PATH_v2 $BASE_PATH_v3; do + if [ -f "/tmp/NullValues_$(basename $BASE).java" ]; then + ORIGINAL_DIR=$(cat "/tmp/NullValues_$(basename $BASE).dir") + mkdir -p "$ORIGINAL_DIR" # Ensure directory exists + cp "/tmp/NullValues_$(basename $BASE).java" "$ORIGINAL_DIR/NullValues.java" + rm "/tmp/NullValues_$(basename $BASE).java" + rm "/tmp/NullValues_$(basename $BASE).dir" + echo "Restored: $ORIGINAL_DIR/NullValues.java" + fi +done diff --git a/java/Dockerfile.build b/java/Dockerfile.build index 9688f76..a734a99 100644 --- a/java/Dockerfile.build +++ b/java/Dockerfile.build @@ -1,4 +1,4 @@ -FROM maven:3.6.3-jdk-8 +FROM maven:3.9.6-eclipse-temurin-17 LABEL org.label-schema.license="Apache-2.0" \ org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \ @@ -9,4 +9,4 @@ COPY java /src RUN /src/CI/build-sdk.sh -ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"] \ No newline at end of file +ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"] diff --git a/java/Dockerfile.generate b/java/Dockerfile.generate index a14814f..001ff61 100644 --- a/java/Dockerfile.generate +++ b/java/Dockerfile.generate @@ -1,8 +1,8 @@ -FROM maven:3.6.3-jdk-8 +FROM maven:3.9.6-eclipse-temurin-17 LABEL org.label-schema.license="Apache-2.0" \ org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \ org.label-schema.description="To generate Java Stach package" \ maintainer="analytics.api.support@factset.com" -ENTRYPOINT ["bash", "/java/CI/generate-sdk.sh"] \ No newline at end of file +ENTRYPOINT ["bash", "/java/CI/generate-sdk.sh"] diff --git a/java/v1/pom.xml b/java/v1/pom.xml index b675faf..e512894 100644 --- a/java/v1/pom.xml +++ b/java/v1/pom.xml @@ -166,4 +166,4 @@ ${version.protobuf} - \ No newline at end of file + diff --git a/python/CI/generate-sdk.sh b/python/CI/generate-sdk.sh index bbeaf59..0a9eb04 100755 --- a/python/CI/generate-sdk.sh +++ b/python/CI/generate-sdk.sh @@ -2,19 +2,30 @@ set -e SCHEMA_PATH=/schema # Location of Protobuf schema -BASE_PATH=/python/fds.protobuf.stach.v3 # Base location of Python generated classes -PACKAGE_PATH=fds/protobuf/stach/v3 +BASE_PATH_V3=/python/fds.protobuf.stach.v3 # Base location of Python generated classes +BASE_PATH_V2=/python/fds.protobuf.stach.v2 # Base location of Python generated classes +PACKAGE_PATH_V3=v3/fds/protobuf/stach/v3 +PACKAGE_PATH_V2=v2/fds/protobuf/stach/v2 -rm -f $BASE_PATH/$PACKAGE_PATH/**/*_pb2.py +# echo Removed old generated code +rm -f $BASE_PATH_V2/$PACKAGE_PATH_V2/**/*_pb2.py +rm -f $BASE_PATH_V3/$PACKAGE_PATH_V3/**/*_pb2.py -echo Removed old generated code +mkdir -p "$BASE_PATH_V3" +mkdir -p "$BASE_PATH_V2" -PROTOFILES=$(find $SCHEMA_PATH/$PACKAGE_PATH -iname "*.proto") -protoc --proto_path $SCHEMA_PATH --python_out $BASE_PATH $PROTOFILES +PROTOFILES=$(find $SCHEMA_PATH/$PACKAGE_PATH_V3 -iname "*.proto") +protoc --proto_path $SCHEMA_PATH/v3 --python_out $BASE_PATH_V3 $PROTOFILES -for f in $(find $BASE_PATH/fds/protobuf/stach/v3 -type d -maxdepth 10); do - cd "$f" - touch __init__.py +for f in $(find $BASE_PATH_V3/fds/protobuf/stach/v3 -type d -maxdepth 10); do + touch "$f/__init__.py" +done; + +PROTOFILES=$(find $SCHEMA_PATH/$PACKAGE_PATH_V2 -iname "*.proto") +protoc --proto_path $SCHEMA_PATH/v2 --python_out $BASE_PATH_V2 $PROTOFILES + +for f in $(find $BASE_PATH_V2/fds/protobuf/stach/v2 -type d -maxdepth 10); do + touch "$f/__init__.py" done; echo Produced new generated code diff --git a/python/Dockerfile.build b/python/Dockerfile.build index 00d500a..ee8c385 100644 --- a/python/Dockerfile.build +++ b/python/Dockerfile.build @@ -1,4 +1,4 @@ -FROM python:3.8 +FROM python:3.9-slim LABEL org.label-schema.license="Apache-2.0" \ org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \ @@ -9,4 +9,4 @@ COPY python /src RUN /src/CI/build-sdk.sh -ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"] \ No newline at end of file +ENTRYPOINT ["bash", "/src/CI/extract-artifacts.sh"] diff --git a/python/Dockerfile.generate b/python/Dockerfile.generate index d8d10f0..4db5039 100644 --- a/python/Dockerfile.generate +++ b/python/Dockerfile.generate @@ -1,4 +1,4 @@ -FROM python:3.8 +FROM python:3.9-slim LABEL org.label-schema.license="Apache-2.0" \ org.label-schema.vcs-url="https://github.com/factset/stachschema-sdks" \ @@ -6,7 +6,8 @@ LABEL org.label-schema.license="Apache-2.0" \ maintainer="analytics.api.support@factset.com" RUN apt-get update && \ - apt-get -y install unzip + apt-get -y install unzip git curl && \ + rm -rf /var/lib/apt/lists/* ARG PROTOC_ZIP=protoc-3.19.4-linux-x86_64.zip RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/$PROTOC_ZIP diff --git a/python/fds.protobuf.stach.v2/setup.py b/python/fds.protobuf.stach.v2/setup.py new file mode 100644 index 0000000..46fea79 --- /dev/null +++ b/python/fds.protobuf.stach.v2/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup, find_packages + +setup(name='fds.protobuf.stach.v2', + version='1.0.3', + description='Stach library in python', + url='', + author='analytics-reporting', + author_email='analytics.api.support@factset.com', + license='Apache License 2.0', + install_requires=["protobuf>=3.19.4"], + packages=find_packages(), + zip_safe=False) diff --git a/python/fds.protobuf.stach.v3/setup.py b/python/fds.protobuf.stach.v3/setup.py index 5dd352e..bd283c6 100644 --- a/python/fds.protobuf.stach.v3/setup.py +++ b/python/fds.protobuf.stach.v3/setup.py @@ -1,12 +1,12 @@ from setuptools import setup, find_packages setup(name='fds.protobuf.stach.v3', - version='1.0.0', + version='1.0.1', description='Stach library in python', url='', author='analytics-reporting', author_email='analytics.api.support@factset.com', license='Apache License 2.0', - install_requires=["protobuf==3.19.4"], + install_requires=["protobuf>=3.19.4"], packages=find_packages(), zip_safe=False)