|
46 | 46 | "id": "MfBg1C5NB3X0" |
47 | 47 | }, |
48 | 48 | "source": [ |
49 | | - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", |
50 | | - " \u003ctd\u003e\n", |
51 | | - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/text/guide/tokenizers\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", |
52 | | - " \u003c/td\u003e\n", |
53 | | - " \u003ctd\u003e\n", |
54 | | - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/text/blob/master/docs/guide/tokenizers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", |
55 | | - " \u003c/td\u003e\n", |
56 | | - " \u003ctd\u003e\n", |
57 | | - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/text/blob/master/docs/guide/tokenizers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView on GitHub\u003c/a\u003e\n", |
58 | | - " \u003c/td\u003e\n", |
59 | | - " \u003ctd\u003e\n", |
60 | | - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/text/docs/guide/tokenizers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", |
61 | | - " \u003c/td\u003e\n", |
62 | | - " \u003ctd\u003e\n", |
63 | | - " \u003ca href=\"https://tfhub.dev/google/zh_segmentation/1\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/hub_logo_32px.png\" /\u003eSee TF Hub models\u003c/a\u003e\n", |
64 | | - " \u003c/td\u003e\n", |
65 | | - "\u003c/table\u003e" |
| 49 | + "<table class=\"tfo-notebook-buttons\" align=\"left\">\n", |
| 50 | + " <td>\n", |
| 51 | + " <a target=\"_blank\" href=\"https://www.tensorflow.org/text/guide/tokenizers\"><img src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" />View on TensorFlow.org</a>\n", |
| 52 | + " </td>\n", |
| 53 | + " <td>\n", |
| 54 | + " <a target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/text/blob/master/docs/guide/tokenizers.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n", |
| 55 | + " </td>\n", |
| 56 | + " <td>\n", |
| 57 | + " <a target=\"_blank\" href=\"https://github.com/tensorflow/text/blob/master/docs/guide/tokenizers.ipynb\"><img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View on GitHub</a>\n", |
| 58 | + " </td>\n", |
| 59 | + " <td>\n", |
| 60 | + " <a href=\"https://storage.googleapis.com/tensorflow_docs/text/docs/guide/tokenizers.ipynb\"><img src=\"https://www.tensorflow.org/images/download_logo_32px.png\" />Download notebook</a>\n", |
| 61 | + " </td>\n", |
| 62 | + " <td>\n", |
| 63 | + " <a href=\"https://tfhub.dev/google/zh_segmentation/1\"><img src=\"https://www.tensorflow.org/images/hub_logo_32px.png\" />See TF Hub models</a>\n", |
| 64 | + " </td>\n", |
| 65 | + "</table>" |
66 | 66 | ] |
67 | 67 | }, |
68 | 68 | { |
|
95 | 95 | }, |
96 | 96 | "outputs": [], |
97 | 97 | "source": [ |
98 | | - "!pip install -q -U tensorflow-text" |
| 98 | + "!pip install -q -U tensorflow-text tensorflow-hub" |
99 | 99 | ] |
100 | 100 | }, |
101 | 101 | { |
|
108 | 108 | "source": [ |
109 | 109 | "import requests\n", |
110 | 110 | "import tensorflow as tf\n", |
| 111 | + "import tensorflow_hub as hub\n", |
111 | 112 | "import tensorflow_text as tf_text" |
112 | 113 | ] |
113 | 114 | }, |
|
438 | 439 | "outputs": [], |
439 | 440 | "source": [ |
440 | 441 | "MODEL_HANDLE = \"https://tfhub.dev/google/zh_segmentation/1\"\n", |
441 | | - "segmenter = tf_text.HubModuleTokenizer(MODEL_HANDLE)\n", |
| 442 | + "segmenter = tf_text.HubModuleTokenizer(hub.resolve(MODEL_HANDLE))\n", |
442 | 443 | "tokens = segmenter.tokenize([\"新华社北京\"])\n", |
443 | 444 | "print(tokens.to_list())" |
444 | 445 | ] |
|
479 | 480 | "source": [ |
480 | 481 | "#### SplitMergeTokenizer\n", |
481 | 482 | "\n", |
482 | | - "The `SplitMergeTokenizer` \u0026 `SplitMergeFromLogitsTokenizer` have a targeted purpose of splitting a string based on provided values that indicate where the string should be split. This is useful when building your own segmentation models like the previous Segmentation example.\n", |
| 483 | + "The `SplitMergeTokenizer` & `SplitMergeFromLogitsTokenizer` have a targeted purpose of splitting a string based on provided values that indicate where the string should be split. This is useful when building your own segmentation models like the previous Segmentation example.\n", |
483 | 484 | "\n", |
484 | 485 | "For the `SplitMergeTokenizer`, a value of 0 is used to indicate the start of a new string, and the value of 1 indicates the character is part of the current string." |
485 | 486 | ] |
|
0 commit comments