Remove OpenNMT and Tensorflow, Update to Python 3.10 (#583)

* Removed OpenNMT and Tensorflow Dependencies

-Removed the memory_growth and eager_execution command line arguments
-Removed WER score metric

* Update Python to 3.10 and transformers to 4.46

Author: TaperChipmunk32

.devcontainer/Dockerfile

@@ -1,6 +1,6 @@
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.10
 ARG POETRY_VERSION=1.7.1
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 ARG PYTHON_VERSION
 ARG POETRY_VERSION
 WORKDIR /app
@@ -44,5 +44,5 @@ ENV SIL_NLP_CACHE_PROJECT_DIR=/root/.cache/silnlp/projects
 # Set environment variables
 ENV CLEARML_API_HOST="https://api.sil.hosted.allegro.ai"
 ENV SIL_NLP_DATA_PATH=/silnlp
-ENV EFLOMAL_PATH=/workspaces/silnlp/.venv/lib/python3.8/site-packages/eflomal/bin
+ENV EFLOMAL_PATH=/workspaces/silnlp/.venv/lib/python3.12/site-packages/eflomal/bin
 CMD ["bash"]

Dockerfile

@@ -1,7 +1,7 @@
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.10
 ARG POETRY_VERSION=1.7.1
 
-FROM python:$PYTHON_VERSION-slim as builder
+FROM python:$PYTHON_VERSION-slim AS builder
 ARG POETRY_VERSION
 
 ENV POETRY_HOME=/opt/poetry
@@ -22,9 +22,9 @@ RUN poetry export -E eflomal --without-hashes -f requirements.txt > requirements
 COPY . /src
 RUN poetry build
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.10
 
 ENV PIP_DISABLE_PIP_VERSION_CHECK=on
 ENV TZ=America/New_York

poetry.lock

@@ -1,6 +1,6 @@
 [tool.black]
 line-length = 120
-target-version = ['py38']
+target-version = ['py310']
 include = '\.pyi?$'
 exclude = '''
 /(
@@ -48,7 +48,7 @@ silnlp-alignment-aggregate-results = "silnlp.alignment.aggregate_results:main"
 clowder = "clowder.clowder:main"
 
 [tool.poetry.dependencies]
-python = ">=3.8.1,<3.9"
+python = ">=3.10,<3.11"
 pandas = "^1.0.4"
 sentencepiece = "^0.1.97"
 nltk = "^3.5"
@@ -57,13 +57,11 @@ seaborn = "0.11.2"
 morfessor = "^2.0.6"
 Morfessor-FlatCat = "^1.0.8"
 psutil = "^5.7.3"
-scikit-learn = "1.1.2"
-numpy = "^1.23.1"
-OpenNMT-tf = "^2.31.0"
+scikit-learn = "^1.1.2"
+numpy = "^1.26.0"
 lit-nlp = "0.4.1"
-tensorflow = "2.7.3"
 google-cloud-translate = "^3.0.2"
-scipy = "1.8"
+scipy = "^1.11.2"
 clearml = ">=1.4.1"
 XlsxWriter = "^3.2.0"
 python-Levenshtein = "^0.20.9"
@@ -72,18 +70,17 @@ tqdm = "^4.62.2"
 s3path = "0.3.4"
 sacrebleu = "^2.3.1"
 ctranslate2 = "^3.5.1"
-tensorflow-addons = "0.17.1"
 libclang = "14.0.6"
-sil-machine = {extras = ["thot"], version = "^1.1.0"}
+sil-machine = {extras = ["thot"], version = "^1.3.0"}
 datasets = "^2.7.1"
-torch = {version = "2.1.2", source = "torch"}
+torch = {version = "^2.4", source = "torch"}
 sacremoses = "^0.0.53"
 evaluate = "^0.3.0"
 python-docx = "^0.8.11"
 iso639-lang = "^2.1.0"
 eflomal = { version = "^2.0.0", optional = true }
-accelerate = "^0.23.0"
-transformers = "^4.36.2"
+accelerate = "^0.26.0"
+transformers = "^4.46"
 optimum = "^1.16.0"
 google = "^3.0.0"
 google-api-python-client = "^2.101.0"

pyproject.toml

@@ -3,6 +3,7 @@
 import numpy as np
 from pathlib import Path
 import matplotlib.pyplot as plt
+import os
 
 LOGGER = logging.getLogger("silnlp")
 

scripts/bible_alignment/scratchpad_align_analyze.py

@@ -5,7 +5,7 @@
     task_name="clear_cache",
 )
 task.set_base_docker(
-    docker_image="nvidia/cuda:11.2.2-cudnn8-runtime-ubuntu20.04",
+    docker_image="nvidia/cuda:11.2.2-cudnn8-runtime-ubuntu22.04",
     docker_arguments="-v /home/clearml/.clearml/hf-cache:/root/.cache/huggingface",
     docker_setup_bash_script=[
         "apt install -y python3-venv",

scripts/clear_clearml_cache.py

@@ -1,9 +1,6 @@
-from silnlp.common.tf_utils import enable_memory_growth
 from silnlp.nmt.experiment import SILExperiment
 from silnlp.nmt.translate import TranslationTask
 
-enable_memory_growth()
-
 exp = SILExperiment(
     name="BT-Swahili/en-swh-1",
     make_stats=True,  # limited by stats_max_size to process only Bibles

scripts/scratchpad_s3.py

@@ -5,32 +5,12 @@
 from typing import List, Optional
 
 import psutil
-from opennmt.utils.wer import sentence_wer
 
 from ..common.corpus import write_corpus
 
 METEOR_FULLY_SUPPORTED_LANGS = {"en", "cz", "de", "es", "fr", "ar"}
 
 
-def compute_wer_score(hyps: List[str], refs: List[List[str]]) -> float:
-    if len(hyps) == 0:
-        return 100.0
-
-    try:
-        wer_score = 0.0
-        for hyp, ref in zip(hyps, refs[0]):
-            wer_score += sentence_wer(ref.lower(), hyp.lower())
-        result = wer_score / len(hyps)
-    except UnicodeDecodeError:
-        print("Unable to compute WER score")
-        result = -1
-    except ZeroDivisionError:
-        print("Cannot divide by zero. Check for empty lines.")
-        result = -1
-
-    return result * 100
-
-
 def compute_meteor_score(lang: str, hyps: List[str], refs: List[List[str]]) -> Optional[float]:
     if lang.lower() not in METEOR_FULLY_SUPPORTED_LANGS:
         return None

silnlp/common/metrics.py

@@ -2,13 +2,14 @@
 Normalization tooling for cleaning up whitespace and punctuation in extract sentences
 See normalize_extracts.py for context
 """
-import logging
-import regex
 
+import logging
 from dataclasses import dataclass
 from enum import Enum, IntEnum
 from typing import Dict, List, Optional, Set, Tuple
 
+import regex
+
 
 class PunctuationCategory(Enum):
     LEFT_CLINGING = "LEFT_CLINGING"
@@ -46,7 +47,7 @@ def shift_slice(slice: StringSlice, offset: int, new_outer: str) -> StringSlice:
     )
 
 
-def find_slices(reg: regex.Regex, text: str) -> List[StringSlice]:
+def find_slices(reg: regex.Pattern, text: str) -> List[StringSlice]:
     return [
         StringSlice(start_index=match.span()[0], end_index=match.span()[1], slice=match.group(), outer=text)
         for match in regex.finditer(reg, text)