Browse Source
Remove incorrect token
Signed-off-by: Jael Gu <mengjia.gu@zilliz.com>
main
1 changed files with
1 additions and
2 deletions
-
codebert.py
|
|
@ -74,8 +74,7 @@ class CodeBert(NNOperator): |
|
|
def __call__(self, txt: str) -> numpy.ndarray: |
|
|
def __call__(self, txt: str) -> numpy.ndarray: |
|
|
try: |
|
|
try: |
|
|
tokens = self.tokenizer.tokenize(txt) |
|
|
tokens = self.tokenizer.tokenize(txt) |
|
|
tokens = [self.tokenizer.cls_token, '<encoder-only>', self.tokenizer.sep_token] + tokens + \ |
|
|
|
|
|
[self.tokenizer.sep_token] |
|
|
|
|
|
|
|
|
tokens = [self.tokenizer.cls_token] + tokens + [self.tokenizer.sep_token] |
|
|
tokens_ids = self.tokenizer.convert_tokens_to_ids(tokens) |
|
|
tokens_ids = self.tokenizer.convert_tokens_to_ids(tokens) |
|
|
inputs = torch.tensor(tokens_ids).unsqueeze(0).to(self.device) |
|
|
inputs = torch.tensor(tokens_ids).unsqueeze(0).to(self.device) |
|
|
except Exception as e: |
|
|
except Exception as e: |
|
|
|
