AI Memory : The Simplest System That Beats Every Complex Solution

Research
4 
import torch
import torch.nn.functional as F
import math

# 問與答
question_words = ["I", "want", "to", "learn", "Python"]
answer_words = ["Start", "with", "basic", "syntax"]

# 建立詞庫映射
vocab = list(set(question_words + answer_words))
word_to_id = {w: i for i, w in enumerate(vocab)}

# 將單詞轉為 id
q_ids = [word_to_id[w] for w in question_words]
a_ids = [word_to_id[w] for w in answer_words]

# 小 embedding 維度
embed_dim = 8
embedding_matrix = torch.randn(len(vocab), embed_dim)

# 取得嵌入向量
q_embed = embedding_matrix[q_ids]
a_embed = embedding_matrix[a_ids]

# 建立 Q, K, V 權重
W_q = torch.randn(embed_dim, embed_dim)
W_k = torch.randn(embed_dim, embed_dim)
W_v = torch.randn(embed_dim, embed_dim)

Q = q_embed @ W_q
K = a_embed @ W_k
V = a_embed @ W_v

# 計算注意力
scores = Q @ K.T / math.sqrt(embed_dim)
attn_weights = F.softmax(scores, dim=-1).detach().numpy()

# 只顯示注意力大於 0.8 的 token pair
threshold = 0.8
high_attention_pairs = []

for i, q_word in enumerate(question_words):
    for j, a_word in enumerate(answer_words):
        score = attn_weights[i][j]
        if score > threshold:
            high_attention_pairs.append((q_word, a_word, score))

# 輸出結果
print("🧠 高注意力 Token Pairs (score > 0.8):")
if high_attention_pairs:
    for q_token, a_token, score in high_attention_pairs:
        print(f"Question: '{q_token}' → Answer: '{a_token}' | Score: {score:.3f}")
else:
    print("沒有任何注意力超過門檻值 0.8")
1 Like