sync update from LAN repo (source ji @ 85078a31)
This commit is contained in:
1 parent
5d6627af50
commit
e602d88695
864 files changed
+99150
-6346
No files matched your search
@@ -234,12 +234,23 @@ async def test_fixed_answer_default_speed_schedules_100_tokens_per_second(
|
||||
async for generation in model._astream([]):
|
||||
chunks.append(str(generation.message.content))
|
||||
|
||||
token_count = len(fixed_answer_module._TOKEN_ENCODING.encode(answer))
|
||||
token_count = len(fixed_answer_module._split_local_tokens(answer))
|
||||
assert "".join(chunks) == answer
|
||||
assert model.tokens_per_second == 100
|
||||
assert sum(sleeps) == pytest.approx((token_count - 1) / 100)
|
||||
|
||||
|
||||
def test_fixed_answer_local_tokenizer_is_lossless_and_offline() -> None:
|
||||
answer = " 离线环境\nEnglish words, 数字 123,emoji 👋🏽。 "
|
||||
|
||||
tokens = fixed_answer_module._split_local_tokens(answer)
|
||||
|
||||
assert "".join(tokens) == answer
|
||||
assert len(tokens) > 1
|
||||
assert len(fixed_answer_module._split_local_tokens("x" * 50)) >= 13
|
||||
assert "tiktoken" not in fixed_answer_module.__dict__
|
||||
|
||||
|
||||
def test_replace_request_rejects_duplicate_exact_questions() -> None:
|
||||
row = {
|
||||
"question": "重复问题",
|
||||
|
||||
Reference in new issue
Block a user