From 7933054638680d545169a066fc330d32fe60927f Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Thu, 12 Nov 2020 11:53:23 -0500 Subject: [PATCH] Model sharing doc (#8498) * Model sharing doc * Style --- docs/source/model_doc/marian.rst | 16 +++++++--- docs/source/model_sharing.rst | 53 +++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/docs/source/model_doc/marian.rst b/docs/source/model_doc/marian.rst index e90123adc4..58d9f4c0a4 100644 --- a/docs/source/model_doc/marian.rst +++ b/docs/source/model_doc/marian.rst @@ -139,11 +139,19 @@ Example of translating english to many romance languages, using old-style 2 char .. code-block::python from transformers import MarianMTModel, MarianTokenizer - src_text = [ '>>fr<< this is a sentence in english that we want to translate to french', '>>pt<< This should go to portuguese', '>>es<< And this to Spanish'] + src_text = [ + '>>fr<< this is a sentence in english that we want to translate to french', + '>>pt<< This should go to portuguese', + '>>es<< And this to Spanish' + ] - model_name = 'Helsinki-NLP/opus-mt-en-ROMANCE' tokenizer = MarianTokenizer.from_pretrained(model_name) - print(tokenizer.supported_language_codes) model = MarianMTModel.from_pretrained(model_name) translated = - model.generate(**tokenizer.prepare_seq2seq_batch(src_text)) tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated] + model_name = 'Helsinki-NLP/opus-mt-en-ROMANCE' + tokenizer = MarianTokenizer.from_pretrained(model_name) + print(tokenizer.supported_language_codes) + + model = MarianMTModel.from_pretrained(model_name) + translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text)) + tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated] # ["c'est une phrase en anglais que nous voulons traduire en français", 'Isto deve ir para o português.', 'Y esto al español'] diff --git a/docs/source/model_sharing.rst b/docs/source/model_sharing.rst index 920fabd871..81f9326163 100644 --- a/docs/source/model_sharing.rst +++ b/docs/source/model_sharing.rst @@ -52,21 +52,21 @@ users to clone it and you (and your organization members) to push to it. First, Go in a terminal and run the following command. It should be in the virtual environment where you installed 🤗 Transformers, since that command :obj:`transformers-cli` comes from the library. -.. code-block:: +.. code-block:: bash transformers-cli login Once you are logged in with your model hub credentials, you can start building your repositories. To create a repo: -.. code-block:: +.. code-block:: bash transformers-cli repo create your-model-name This creates a repo on the model hub, which can be cloned. You can then add/remove from that repo as you would with any other git repo. -.. code-block:: +.. code-block:: bash git clone https://huggingface.co/username/your-model-name @@ -159,24 +159,25 @@ Or, if you're using the Trainer API .. code-block:: >>> trainer.save_model("path/to/awesome-name-you-picked") + >>> tokenizer.save_pretrained("path/to/repo/clone/your-model-name") You can then add these files to the staging environment and verify that they have been correctly staged with the ``git status`` command: -.. code-block:: +.. code-block:: bash git add --all git status Finally, the files should be comitted: -.. code-block:: +.. code-block:: bash git commit -m "First version of the your-model-name model and tokenizer." And pushed to the remote: -.. code-block:: +.. code-block:: bash git push @@ -199,7 +200,7 @@ don't forget to link to its model card so that people can fully trace how your m If you have never made a pull request to the 🤗 Transformers repo, look at the :doc:`contributing guide ` to see the steps to follow. -.. Note:: +.. note:: You can also send your model card in the folder you uploaded with the CLI by placing it in a `README.md` file inside `path/to/awesome-name-you-picked/`. @@ -225,3 +226,41 @@ You may specify a revision by using the ``revision`` flag in the ``from_pretrain >>> "julien-c/EsperBERTo-small", >>> revision="v2.0.1" # tag name, or branch name, or commit hash >>> ) + +Workflow in a Colab notebook +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you're in a Colab notebook (or similar) with no direct access to a terminal, here is the workflow you can use to +upload your model. You can execute each one of them in a cell by adding a ! at the beginning. + +First you need to install `git-lfs` in the environment used by the notebook: + +.. code-block:: bash + + sudo apt-get install git-lfs + +Then you can use the :obj:`transformers-cli` to create your new repo: + + +.. code-block:: bash + + transformers-cli login + transformers-cli repo create your-model-name + +Once it's created, you can clone it and configure it (replace username by your username on huggingface.co): + +.. code-block:: bash + + git clone https://huggingface.co/username/your-model-name + cd your-model-name + git lfs install + git config --global user.email "email@example.com" + +Once you've saved your model inside, you can add it and push it with usual git commands. Note that you have to replace +`username:password` with your username and password to huggingface.co. + +.. code-block:: bash + + git add . + git commit -m "Initial commit" + git push https://username:password@huggingface.co/username/your-model-name