Skip to content

Commit 889d74a

Browse files
karllessardkarllessard
committed
Fix/javadoc script (#555)
1 parent 9cf8693 commit 889d74a

File tree

4 files changed

+150
-34
lines changed

4 files changed

+150
-34
lines changed

‎.github/workflows/build.yml

+23-24
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,28 @@ jobs:
6262
fi
6363
echo "Repository URL: $REPOSITORY_URL"
6464
echo "::set-output name=repositoryUrl::$REPOSITORY_URL"
65+
# linux-arm64:
66+
# runs-on: linux-arm64-ubuntu2204
67+
# needs: prepare
68+
# strategy:
69+
# matrix:
70+
# ext: [""]
71+
# steps:
72+
# - name: Install environment
73+
# run: |
74+
# sudo apt update
75+
# sudo apt install -y curl wget unzip tar git gcc g++ maven default-jdk
76+
# - name: Checkout repository
77+
# uses: actions/checkout@v1
78+
# - name: Build project
79+
# run: |
80+
# gcc --version
81+
# mvn -version
82+
# echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml
83+
# mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }}
84+
# - name: Deploy native artifact
85+
# if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true'
86+
# run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }}
6587
linux-x86_64:
6688
runs-on: ubuntu-20.04
6789
needs: prepare
@@ -178,32 +200,9 @@ jobs:
178200
run: |
179201
call mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }}
180202
if ERRORLEVEL 1 exit /b
181-
linux-arm64:
182-
runs-on: linux-arm64-ubuntu2204
183-
needs: prepare
184-
strategy:
185-
matrix:
186-
ext: [""]
187-
steps:
188-
- name: Install environment
189-
run: |
190-
sudo apt update
191-
sudo apt install -y curl wget unzip tar git gcc g++ maven default-jdk
192-
- name: Checkout repository
193-
uses: actions/checkout@v1
194-
- name: Build project
195-
run: |
196-
gcc --version
197-
mvn -version
198-
echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml
199-
mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }}
200-
- name: Deploy native artifact
201-
if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true'
202-
run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }}
203-
204203
deploy:
205204
if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging') }} # DEPLOY_SNAPSHOT (releases should be signed and deployed manually from local machine)
206-
needs: [linux-x86_64, macosx-x86_64, windows-x86_64, macosx-arm64, linux-arm64]
205+
needs: [linux-x86_64, macosx-x86_64, windows-x86_64, macosx-arm64] #, linux-arm64]
207206
runs-on: ubuntu-20.04
208207
steps:
209208
- name: Configure Java

‎tensorflow-core/tensorflow-core-native/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -669,6 +669,7 @@
669669
<minmemory>256m</minmemory>
670670
<maxmemory>2048m</maxmemory>
671671
<links>
672+
<link>https://protobuf.dev/reference/java/api-docs</link>
672673
<link>http://bytedeco.org/javacpp/apidocs</link>
673674
</links>
674675
</configuration>

‎tools/build_java_api_docs.py

+28-10
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,23 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515
# ==============================================================================
16-
"""Generate TensorFlow Lite Java reference docs for TensorFlow.org."""
16+
"""Generate TensorFlow Java reference docs for TensorFlow.org."""
1717
from __future__ import absolute_import
1818
from __future__ import division
1919
from __future__ import print_function
2020

2121
import pathlib
2222
import shutil
2323
import tempfile
24+
from git import Repo
2425

2526
from absl import app
2627
from absl import flags
2728

2829
from tensorflow_docs.api_generator import gen_java
2930

3031
FLAGS = flags.FLAGS
32+
NDARRAY_VERSION = 'v1.0.0'
3133

3234
# These flags are required by infrastructure, not all of them are used.
3335
flags.DEFINE_string('output_dir', '/tmp/java_api/',
@@ -48,6 +50,18 @@
4850
TOOLS_DIR = pathlib.Path(__file__).resolve().parent
4951
REPO_ROOT = TOOLS_DIR.parent
5052

53+
54+
def checkout_ndarray():
55+
repo_url = 'https://github.com/tensorflow/java-ndarray'
56+
local_repo_path = REPO_ROOT/'ndarray'
57+
if not pathlib.Path(local_repo_path).exists():
58+
local_repo = Repo.clone_from(repo_url, local_repo_path)
59+
else:
60+
local_repo = Repo(local_repo_path)
61+
local_repo.remotes['origin'].fetch()
62+
local_repo.git.checkout(NDARRAY_VERSION)
63+
64+
5165
def overlay(from_root, to_root):
5266
for from_path in pathlib.Path(from_root).rglob('*'):
5367
relpath = from_path.relative_to(from_root)
@@ -58,24 +72,28 @@ def overlay(from_root, to_root):
5872
else:
5973
to_path.mkdir(exist_ok=True)
6074

75+
6176
def main(unused_argv):
77+
checkout_ndarray()
6278
merged_source = pathlib.Path(tempfile.mkdtemp())
6379
(merged_source / 'java/org').mkdir(parents=True)
6480

65-
shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/',
66-
merged_source/'java/org/tensorflow')
67-
overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow',
68-
merged_source/'java/org/tensorflow')
69-
shutil.copytree(REPO_ROOT/'tensorflow-framework/src/main/java/org/tensorflow/framework',
70-
merged_source/'java/org/tensorflow/framework')
71-
shutil.copytree(REPO_ROOT/'ndarray/src/main/java/org/tensorflow/ndarray',
72-
merged_source/'java/org/tensorflow/ndarray')
81+
shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/', merged_source/'java/org/tensorflow')
82+
overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow', merged_source/'java/org/tensorflow')
83+
shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/proto', merged_source/'java/org/tensorflow/proto')
84+
shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/exceptions', merged_source/'java/org/tensorflow/exceptions')
85+
shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api', merged_source/'java/org/tensorflow/internal/c_api')
86+
shutil.copytree(REPO_ROOT/'tensorflow-framework/src/main/java/org/tensorflow/framework', merged_source/'java/org/tensorflow/framework')
87+
shutil.copytree(REPO_ROOT/'ndarray/ndarray/src/main/java/org/tensorflow/ndarray', merged_source/'java/org/tensorflow/ndarray')
7388

7489
gen_java.gen_java_docs(
7590
package='org.tensorflow',
7691
source_path=merged_source / 'java',
7792
output_dir=pathlib.Path(FLAGS.output_dir),
78-
site_path=pathlib.Path(FLAGS.site_path))
93+
site_path=pathlib.Path(FLAGS.site_path),
94+
# Uncomment for local testing:
95+
# script_path=pathlib.Path(REPO_ROOT/'tools/run-javadoc-for-tf-local.sh'),
96+
)
7997

8098

8199
if __name__ == '__main__':

‎tools/run-javadoc-for-tf-local.sh

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/bin/bash
2+
set -ex
3+
4+
export JAVA_HOME=/Library/Java/JavaVirtualMachines/zulu-11.jdk/Contents/Home # Or change to any JDK 11 home path
5+
6+
# https://android.googlesource.com/platform/external/doclava/
7+
# There's a debian package:
8+
# https://packages.debian.org/unstable/doclava-aosp
9+
# Install with:
10+
#
11+
# $ sudo apt install doclava-aosp #v 6.0.1+r55-1+build1
12+
#
13+
# https://unix.stackexchange.com/questions/594841/how-do-i-assign-a-value-to-a-bash-variable-iff-that-variable-is-null-unassigned
14+
DOCLAVA_JAR=${DOCLAVA_JAR:-'lib/doclava.jar'} # Build lib locally
15+
16+
# Install java clear silver templates with:
17+
#
18+
# $ sudo apt install libjsilver-aosp-java #v 6.0.1+r55-1+build1
19+
JSILVER_JAR=${JSILVER_JAR:-'lib/jsilver.jar'} # Build lib locally
20+
21+
22+
######### DELETE OUTPUT_DIR #################
23+
24+
# Empty the output directory in case a class has been deleted
25+
rm -rf "${OUTPUT_DIR:?}"/*
26+
############ RUN DOCLAVA ###################
27+
28+
# $FEDERATED_DOCS is a space-separated string of url,file pairs.
29+
read -a api_pairs <<< "${FEDERATED_DOCS}"
30+
FEDERATED_PARAMS=""
31+
for i in "${!api_pairs[@]}"; do
32+
api_pair_str="${api_pairs[$i]}" # "url,api.txt"
33+
read -a api_pair <<< "${api_pair_str//,/ }"
34+
# Using the index as the API "name", build the federation params. Note that
35+
# using 0 as an API name will evaluate to false and cause rendering bugs,
36+
# so we preface with "api_".
37+
FEDERATED_PARAMS+=" -federate api_${i} ${api_pair[0]}"
38+
FEDERATED_PARAMS+=" -federationapi api_${i} ${api_pair[1]}"
39+
done
40+
41+
# To install javadoc, for example, use
42+
#
43+
# sudo apt install openjdk-11-jdk
44+
#
45+
# doclava doesn't work with openjdk-13
46+
# ```
47+
# javadoc: error - Class com.google.doclava.Doclava is not a valid doclet.
48+
# Note: As of JDK 13, the com.sun.javadoc API is no longer supported.
49+
# ```
50+
# It's used here: https://android.googlesource.com/platform/external/doclava/+/refs/heads/master/src/com/google/doclava/Doclava.java
51+
52+
# Each package in $PACKAGE needs to prefaced with -subpackages, so do that.
53+
SUBPACKAGES=""
54+
read -r -a packages <<< "${PACKAGE}"
55+
for pkg in "${packages[@]}"; do
56+
SUBPACKAGES+=" -subpackages ${pkg}"
57+
done
58+
( # Capture the return code. it may be non-zero for minor errors.
59+
javadoc \
60+
-sourcepath "${SOURCE_PATH}" \
61+
-docletpath "${DOCLAVA_JAR}:${JSILVER_JAR}" \
62+
-doclet com.google.doclava.Doclava \
63+
-d "${OUTPUT_DIR}" \
64+
-toroot "${SITE_PATH}"/ \
65+
-yaml _toc.yaml \
66+
-templatedir "${TEMPLATES}" \
67+
-public \
68+
-devsite \
69+
${FEDERATED_PARAMS} \
70+
${SUBPACKAGES}
71+
)
72+
73+
74+
mv "${OUTPUT_DIR}"/reference/* "${OUTPUT_DIR}"
75+
76+
###################################################################
77+
################### START OF POST-PROCESSING ######################
78+
###################################################################
79+
rm "${OUTPUT_DIR}/navtree_data.js" || true
80+
rm "${OUTPUT_DIR}/hierarchy.html" || true
81+
82+
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|${SITE_PATH}/reference|${SITE_PATH}|g"
83+
find ${OUTPUT_DIR} -name "*.yaml" | xargs sed -i '' "s|${SITE_PATH}/reference|${SITE_PATH}|g"
84+
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/org/tensorflow|a href=\"${SITE_PATH}/org/tensorflow|g"
85+
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/com/google|a href=\"${SITE_PATH}/com/google|g"
86+
87+
JAVA_LANG=https://docs.oracle.com/javase/8/docs/api
88+
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/java/lang|a href=\"${JAVA_LANG}/java/lang|g"
89+
90+
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' 's|<pre><code>|<pre class="prettyprint"><code>|g'
91+
92+
rm ${OUTPUT_DIR}/timestamp.js || true
93+
rm ${OUTPUT_DIR}/lists.js || true
94+
rm ${OUTPUT_DIR}/index.html || true
95+
96+
cp ${TEMPLATES}/screen.css ${OUTPUT_DIR}/
97+
98+

0 commit comments

Comments
 (0)