phase 5 part 5: cs algorithms, network-os, web-security topics with tested examples; all 43 stacks pass --strict
This commit is contained in:
Shikui Wang committed
1 parent
f695bbd7cc
commit
691f738f5f
22 files changed
+1955
-37
No files matched your search
@@ -0,0 +1,92 @@
|
||||
"""TCP 是字节流：演示“粘包 / 拆包”现象，以及用“长度前缀”协议正确分帧。只用标准库，`python framing.py` 运行。
|
||||
|
||||
TCP 只保证字节按顺序可靠到达，不保留“消息边界”：发送方 send 了 3 次，接收方可能 1 次 recv 全部收到，
|
||||
也可能把一条消息分成多次收到。应用层必须自己定义边界：换行分隔（如 SSE、JSON Lines）或长度前缀（如很多 RPC 协议）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import socket
|
||||
import struct
|
||||
import threading
|
||||
|
||||
HEADER = struct.Struct("!I") # 4 字节无符号整数，网络字节序（大端）
|
||||
|
||||
|
||||
def encode(msg: dict) -> bytes:
|
||||
body = json.dumps(msg, ensure_ascii=False).encode()
|
||||
return HEADER.pack(len(body)) + body
|
||||
|
||||
|
||||
class FrameDecoder:
|
||||
"""增量解码：不管数据被怎样切分送进来，都能还原出完整的消息。"""
|
||||
|
||||
def __init__(self, max_frame: int = 1 << 20):
|
||||
self.buf = bytearray()
|
||||
self.max_frame = max_frame
|
||||
|
||||
def feed(self, data: bytes) -> list[dict]:
|
||||
self.buf += data
|
||||
out = []
|
||||
while len(self.buf) >= HEADER.size:
|
||||
(n,) = HEADER.unpack_from(self.buf)
|
||||
if n > self.max_frame:
|
||||
raise ValueError(f"帧长度 {n} 超过上限，可能是攻击或协议错误")
|
||||
if len(self.buf) < HEADER.size + n:
|
||||
break # 数据还不完整，等下一次
|
||||
body = bytes(self.buf[HEADER.size:HEADER.size + n])
|
||||
del self.buf[:HEADER.size + n]
|
||||
out.append(json.loads(body))
|
||||
return out
|
||||
|
||||
|
||||
def demo_sticky() -> int:
|
||||
"""连续 send 三次小消息，看接收端一次 recv 拿到几段。"""
|
||||
srv = socket.create_server(("127.0.0.1", 0))
|
||||
port = srv.getsockname()[1]
|
||||
received: list[bytes] = []
|
||||
|
||||
def server() -> None:
|
||||
conn, _ = srv.accept()
|
||||
with conn:
|
||||
threading.Event().wait(0.2) # 等发送方把三条都发完
|
||||
received.append(conn.recv(4096))
|
||||
|
||||
t = threading.Thread(target=server)
|
||||
t.start()
|
||||
with socket.create_connection(("127.0.0.1", port)) as c:
|
||||
for i in range(3):
|
||||
c.sendall(f"msg{i}".encode())
|
||||
t.join()
|
||||
srv.close()
|
||||
return len(received[0])
|
||||
|
||||
|
||||
def demo_framed() -> list[dict]:
|
||||
srv = socket.create_server(("127.0.0.1", 0))
|
||||
port = srv.getsockname()[1]
|
||||
got: list[dict] = []
|
||||
|
||||
def server() -> None:
|
||||
conn, _ = srv.accept()
|
||||
dec = FrameDecoder()
|
||||
with conn:
|
||||
while chunk := conn.recv(3): # 故意每次只读 3 字节，模拟拆包
|
||||
got.extend(dec.feed(chunk))
|
||||
|
||||
t = threading.Thread(target=server)
|
||||
t.start()
|
||||
with socket.create_connection(("127.0.0.1", port)) as c:
|
||||
c.sendall(b"".join(encode({"seq": i, "text": f"第{i}条"}) for i in range(3))) # 一次发出，模拟粘包
|
||||
t.join()
|
||||
srv.close()
|
||||
return got
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
n = demo_sticky()
|
||||
note = "三条消息粘在一起，边界丢失" if n > 4 else "这次恰好没有粘包，但不能依赖这一点"
|
||||
print(f"发送方 send 了 3 次（每次 4 字节），接收方一次 recv 收到 {n} 字节 → {note}")
|
||||
msgs = demo_framed()
|
||||
assert [m["seq"] for m in msgs] == [0, 1, 2], msgs
|
||||
print(f"长度前缀分帧：每次只读 3 字节，仍然还原出 {len(msgs)} 条完整消息：{[m['text'] for m in msgs]}")
|
||||
@@ -0,0 +1,54 @@
|
||||
"""I/O 多路复用：单线程用 selectors（Linux 上是 epoll）同时服务多个连接的回显服务器。只用标准库，`python multiplex_server.py` 运行自检。
|
||||
|
||||
对比“一个连接一个线程”：多路复用由内核告诉我们哪些连接可读，一个线程就能处理大量空闲连接。
|
||||
Nginx、Redis、Node.js、Python asyncio 的事件循环都建立在这个机制之上。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import selectors
|
||||
import socket
|
||||
import threading
|
||||
|
||||
|
||||
def serve(srv: socket.socket, stop: threading.Event) -> int:
|
||||
sel = selectors.DefaultSelector()
|
||||
srv.setblocking(False)
|
||||
sel.register(srv, selectors.EVENT_READ, data=None)
|
||||
max_clients = 0
|
||||
while not stop.is_set():
|
||||
for key, _ in sel.select(timeout=0.05):
|
||||
if key.data is None: # 监听套接字可读 = 有新连接
|
||||
conn, _ = srv.accept()
|
||||
conn.setblocking(False)
|
||||
sel.register(conn, selectors.EVENT_READ, data="client")
|
||||
max_clients = max(max_clients, len(sel.get_map()) - 1)
|
||||
else:
|
||||
conn = key.fileobj
|
||||
data = conn.recv(1024) # type: ignore[union-attr]
|
||||
if data:
|
||||
conn.sendall(data.upper()) # type: ignore[union-attr] # 示例简化：小数据量下 sendall 不会阻塞太久
|
||||
else: # 对端关闭
|
||||
sel.unregister(conn)
|
||||
conn.close() # type: ignore[union-attr]
|
||||
sel.close()
|
||||
return max_clients
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
srv = socket.create_server(("127.0.0.1", 0))
|
||||
port = srv.getsockname()[1]
|
||||
stop = threading.Event()
|
||||
result: list[int] = []
|
||||
t = threading.Thread(target=lambda: result.append(serve(srv, stop)))
|
||||
t.start()
|
||||
clients = [socket.create_connection(("127.0.0.1", port)) for _ in range(50)] # 50 个连接同时在线
|
||||
for i, c in enumerate(clients):
|
||||
c.sendall(f"hi{i}".encode())
|
||||
replies = [c.recv(1024).decode() for c in clients]
|
||||
for c in clients:
|
||||
c.close()
|
||||
stop.set()
|
||||
t.join()
|
||||
srv.close()
|
||||
assert replies == [f"HI{i}" for i in range(50)]
|
||||
print(f"单线程同时服务 {result[0]} 个连接，全部回显正确")
|
||||
@@ -0,0 +1,23 @@
|
||||
import random
|
||||
|
||||
import pytest
|
||||
|
||||
from framing import FrameDecoder, encode
|
||||
|
||||
|
||||
def test_any_split_restores_messages():
|
||||
msgs = [{"seq": i, "text": "你好" * i} for i in range(20)]
|
||||
stream = b"".join(encode(m) for m in msgs)
|
||||
rng = random.Random(42)
|
||||
for _ in range(100): # 随机切分 100 种方式
|
||||
dec, out, i = FrameDecoder(), [], 0
|
||||
while i < len(stream):
|
||||
j = i + rng.randint(1, 17)
|
||||
out += dec.feed(stream[i:j])
|
||||
i = j
|
||||
assert out == msgs
|
||||
|
||||
|
||||
def test_rejects_oversized_frame():
|
||||
with pytest.raises(ValueError):
|
||||
FrameDecoder(max_frame=10).feed(encode({"text": "x" * 100}))
|
||||
Reference in new issue
Block a user