Fix/javadoc script

.github/workflows/build.yml

@@ -62,6 +62,28 @@ jobs:
           fi
           echo "Repository URL: $REPOSITORY_URL"
           echo "::set-output name=repositoryUrl::$REPOSITORY_URL"
+#  linux-arm64:
+#    runs-on: linux-arm64-ubuntu2204
+#    needs: prepare
+#    strategy:
+#      matrix:
+#        ext: [""]
+#    steps:
+#      - name: Install environment
+#        run: |
+#          sudo apt update
+#          sudo apt install -y curl wget unzip tar git gcc g++ maven default-jdk
+#      - name: Checkout repository
+#        uses: actions/checkout@v1
+#      - name: Build project
+#        run: |
+#          gcc --version
+#          mvn -version
+#          echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml
+#          mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }}
+#      - name: Deploy native artifact
+#        if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true'
+#        run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }}
   linux-x86_64:
     runs-on: ubuntu-20.04
     needs: prepare
@@ -178,32 +200,9 @@ jobs:
         run: |
           call mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }}
           if ERRORLEVEL 1 exit /b
-  linux-arm64:
-    runs-on: linux-arm64-ubuntu2204
-    needs: prepare
-    strategy:
-       matrix:
-         ext: [""]
-    steps:
-      - name: Install environment
-        run: |
-          sudo apt update
-          sudo apt install -y curl wget unzip tar git gcc g++ maven default-jdk
-      - name: Checkout repository
-        uses: actions/checkout@v1
-      - name: Build project
-        run: |
-          gcc --version
-          mvn -version
-          echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml
-          mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }}
-      - name: Deploy native artifact
-        if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true'
-        run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }}
-
   deploy:
     if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging') }} # DEPLOY_SNAPSHOT (releases should be signed and deployed manually from local machine)
-    needs: [linux-x86_64, macosx-x86_64, windows-x86_64, macosx-arm64, linux-arm64]
+    needs: [linux-x86_64, macosx-x86_64, windows-x86_64, macosx-arm64] #, linux-arm64]
     runs-on: ubuntu-20.04
     steps:
       - name: Configure Java

tensorflow-core/tensorflow-core-native/pom.xml

@@ -669,6 +669,7 @@
               <minmemory>256m</minmemory>
               <maxmemory>2048m</maxmemory>
               <links>
+                <link>https://protobuf.dev/reference/java/api-docs</link>
                 <link>http://bytedeco.org/javacpp/apidocs</link>
               </links>
             </configuration>

tools/build_java_api_docs.py

@@ -13,21 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Generate TensorFlow Lite Java reference docs for TensorFlow.org."""
+"""Generate TensorFlow Java reference docs for TensorFlow.org."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
 import pathlib
 import shutil
 import tempfile
+from git import Repo
 
 from absl import app
 from absl import flags
 
 from tensorflow_docs.api_generator import gen_java
 
 FLAGS = flags.FLAGS
+NDARRAY_VERSION = 'v1.0.0'
 
 # These flags are required by infrastructure, not all of them are used.
 flags.DEFINE_string('output_dir', '/tmp/java_api/',
@@ -48,6 +50,18 @@
 TOOLS_DIR = pathlib.Path(__file__).resolve().parent
 REPO_ROOT = TOOLS_DIR.parent
 
+
+def checkout_ndarray():
+  repo_url = 'https://github.com/tensorflow/java-ndarray'
+  local_repo_path = REPO_ROOT/'ndarray'
+  if not pathlib.Path(local_repo_path).exists():
+    local_repo = Repo.clone_from(repo_url, local_repo_path)
+  else:
+    local_repo = Repo(local_repo_path)
+  local_repo.remotes['origin'].fetch()
+  local_repo.git.checkout(NDARRAY_VERSION)
+
+
 def overlay(from_root, to_root):
   for from_path in pathlib.Path(from_root).rglob('*'):
     relpath = from_path.relative_to(from_root)
@@ -58,24 +72,28 @@ def overlay(from_root, to_root):
     else:
       to_path.mkdir(exist_ok=True)
 
+
 def main(unused_argv):
+  checkout_ndarray()
   merged_source = pathlib.Path(tempfile.mkdtemp())
   (merged_source / 'java/org').mkdir(parents=True)
 
-  shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/',
-                  merged_source/'java/org/tensorflow')
-  overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow',
-            merged_source/'java/org/tensorflow')
-  shutil.copytree(REPO_ROOT/'tensorflow-framework/src/main/java/org/tensorflow/framework',
-                  merged_source/'java/org/tensorflow/framework')
-  shutil.copytree(REPO_ROOT/'ndarray/src/main/java/org/tensorflow/ndarray',
-                  merged_source/'java/org/tensorflow/ndarray')
+  shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/', merged_source/'java/org/tensorflow')
+  overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow', merged_source/'java/org/tensorflow')
+  shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/proto', merged_source/'java/org/tensorflow/proto')
+  shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/exceptions', merged_source/'java/org/tensorflow/exceptions')
+  shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api', merged_source/'java/org/tensorflow/internal/c_api')
+  shutil.copytree(REPO_ROOT/'tensorflow-framework/src/main/java/org/tensorflow/framework', merged_source/'java/org/tensorflow/framework')
+  shutil.copytree(REPO_ROOT/'ndarray/ndarray/src/main/java/org/tensorflow/ndarray', merged_source/'java/org/tensorflow/ndarray')
 
   gen_java.gen_java_docs(
       package='org.tensorflow',
       source_path=merged_source / 'java',
       output_dir=pathlib.Path(FLAGS.output_dir),
-      site_path=pathlib.Path(FLAGS.site_path))
+      site_path=pathlib.Path(FLAGS.site_path),
+      # Uncomment for local testing:
+      # script_path=pathlib.Path(REPO_ROOT/'tools/run-javadoc-for-tf-local.sh'),
+  )
 
 
 if __name__ == '__main__':

tools/run-javadoc-for-tf-local.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+set -ex
+
+export JAVA_HOME=/Library/Java/JavaVirtualMachines/zulu-11.jdk/Contents/Home # Or change to any JDK 11 home path
+
+# https://android.googlesource.com/platform/external/doclava/
+# There's a debian package:
+# https://packages.debian.org/unstable/doclava-aosp
+# Install with:
+#
+#   $ sudo apt install doclava-aosp #v 6.0.1+r55-1+build1
+#
+# https://unix.stackexchange.com/questions/594841/how-do-i-assign-a-value-to-a-bash-variable-iff-that-variable-is-null-unassigned
+DOCLAVA_JAR=${DOCLAVA_JAR:-'lib/doclava.jar'} # Build lib locally
+
+# Install java clear silver templates with:
+#
+#   $ sudo apt install libjsilver-aosp-java #v 6.0.1+r55-1+build1
+JSILVER_JAR=${JSILVER_JAR:-'lib/jsilver.jar'} # Build lib locally
+
+
+######### DELETE OUTPUT_DIR #################
+
+# Empty the output directory in case a class has been deleted
+rm -rf "${OUTPUT_DIR:?}"/*
+############ RUN DOCLAVA ###################
+
+# $FEDERATED_DOCS is a space-separated string of url,file pairs.
+read -a api_pairs <<< "${FEDERATED_DOCS}"
+FEDERATED_PARAMS=""
+for i in "${!api_pairs[@]}"; do
+  api_pair_str="${api_pairs[$i]}"  # "url,api.txt"
+  read -a api_pair <<< "${api_pair_str//,/ }"
+  # Using the index as the API "name", build the federation params. Note that
+  # using 0 as an API name will evaluate to false and cause rendering bugs,
+  # so we preface with "api_".
+  FEDERATED_PARAMS+=" -federate api_${i} ${api_pair[0]}"
+  FEDERATED_PARAMS+=" -federationapi api_${i} ${api_pair[1]}"
+done
+
+# To install javadoc, for example, use
+#
+#   sudo apt install openjdk-11-jdk
+#
+# doclava doesn't work with openjdk-13
+#   ```
+#   javadoc: error - Class com.google.doclava.Doclava is not a valid doclet.
+#   Note: As of JDK 13, the com.sun.javadoc API is no longer supported.
+#   ```
+#   It's used here: https://android.googlesource.com/platform/external/doclava/+/refs/heads/master/src/com/google/doclava/Doclava.java
+
+# Each package in $PACKAGE needs to prefaced with -subpackages, so do that.
+SUBPACKAGES=""
+read -r -a packages <<< "${PACKAGE}"
+for pkg in "${packages[@]}"; do
+  SUBPACKAGES+=" -subpackages ${pkg}"
+done
+( # Capture the return code. it may be non-zero for minor errors.
+  javadoc \
+  -sourcepath "${SOURCE_PATH}" \
+  -docletpath "${DOCLAVA_JAR}:${JSILVER_JAR}" \
+  -doclet com.google.doclava.Doclava \
+  -d "${OUTPUT_DIR}" \
+  -toroot "${SITE_PATH}"/ \
+  -yaml _toc.yaml \
+  -templatedir "${TEMPLATES}" \
+  -public \
+  -devsite \
+  ${FEDERATED_PARAMS} \
+  ${SUBPACKAGES}
+)
+
+
+mv "${OUTPUT_DIR}"/reference/* "${OUTPUT_DIR}"
+
+###################################################################
+################### START OF POST-PROCESSING ######################
+###################################################################
+rm "${OUTPUT_DIR}/navtree_data.js" || true
+rm "${OUTPUT_DIR}/hierarchy.html" || true
+
+find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|${SITE_PATH}/reference|${SITE_PATH}|g"
+find ${OUTPUT_DIR} -name "*.yaml" | xargs sed -i '' "s|${SITE_PATH}/reference|${SITE_PATH}|g"
+find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/org/tensorflow|a href=\"${SITE_PATH}/org/tensorflow|g"
+find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/com/google|a href=\"${SITE_PATH}/com/google|g"
+
+JAVA_LANG=https://docs.oracle.com/javase/8/docs/api
+find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/java/lang|a href=\"${JAVA_LANG}/java/lang|g"
+
+find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' 's|<pre><code>|<pre class="prettyprint"><code>|g'
+
+rm ${OUTPUT_DIR}/timestamp.js || true
+rm ${OUTPUT_DIR}/lists.js || true
+rm ${OUTPUT_DIR}/index.html || true
+
+cp ${TEMPLATES}/screen.css ${OUTPUT_DIR}/
+
+