215 lines
8.7 KiB
Python
215 lines
8.7 KiB
Python
"""
|
|
classifier_dataset.load() 로 데이터 로드 후 TF-IDF + SVM 학습
|
|
"""
|
|
import random
|
|
|
|
from collections import Counter
|
|
|
|
from loguru import logger
|
|
from sklearn.calibration import CalibratedClassifierCV
|
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
from sklearn.metrics import classification_report
|
|
from sklearn.model_selection import train_test_split
|
|
from sklearn.pipeline import Pipeline
|
|
from sklearn.svm import LinearSVC
|
|
|
|
from config import SERVICE
|
|
from train import classifier_dataset
|
|
from utils import classifier_model_store
|
|
|
|
_status: dict = {"status": "idle", "message": ""}
|
|
|
|
|
|
def get_status() -> dict:
|
|
return _status
|
|
|
|
|
|
def augment_text(text: str, n: int = 25) -> list:
|
|
"""텍스트 증강 - 단어 순서 변경, 일부 단어 제거"""
|
|
words = text.split()
|
|
augmented = []
|
|
|
|
for _ in range(n):
|
|
aug = words.copy()
|
|
|
|
# 1. 랜덤 단어 10% 제거
|
|
drop_count = max(1, int(len(aug) * 0.1))
|
|
for _ in range(drop_count):
|
|
if aug:
|
|
aug.pop(random.randint(0, len(aug) - 1))
|
|
|
|
# 2. 랜덤 단어 일부 순서 섞기
|
|
if len(aug) > 5:
|
|
idx = random.randint(0, len(aug) - 3)
|
|
aug[idx], aug[idx + 1] = aug[idx + 1], aug[idx]
|
|
|
|
augmented.append(" ".join(aug))
|
|
|
|
return augmented
|
|
|
|
|
|
def run_train() -> None:
|
|
global _status
|
|
try:
|
|
_status = {"status": "training", "message": "데이터셋 로딩 중..."}
|
|
|
|
data = classifier_dataset.load()
|
|
if not data:
|
|
raise ValueError("데이터셋이 없습니다. /classifier/dataset/build 먼저 실행하세요.")
|
|
|
|
texts = [d["text"] for d in data]
|
|
labels = [d["label"] for d in data]
|
|
|
|
# ── 특정 클래스 데이터 증강 ──────────────────────
|
|
if SERVICE == "amko":
|
|
AUGMENT_LABELS = ["AMKOR TECHNOLOGY TAIWAN_B"] # 증강할 클래스
|
|
AUGMENT_TARGET = 30 # 목표 샘플 수
|
|
|
|
augmented_texts, augmented_labels = [], []
|
|
label_groups = {}
|
|
for t, l in zip(texts, labels):
|
|
label_groups.setdefault(l, []).append(t)
|
|
|
|
for label in AUGMENT_LABELS:
|
|
samples = label_groups.get(label, [])
|
|
current_count = len(samples)
|
|
|
|
if current_count < AUGMENT_TARGET:
|
|
need = AUGMENT_TARGET - current_count
|
|
logger.info(f"{label}: {current_count}개 → {AUGMENT_TARGET}개 증강 ({need}개 추가)")
|
|
|
|
for i in range(need):
|
|
base_text = samples[i % current_count] # 기존 샘플 순환
|
|
aug = augment_text(base_text, n=1)[0]
|
|
augmented_texts.append(aug)
|
|
augmented_labels.append(label)
|
|
|
|
texts = texts + augmented_texts
|
|
labels = labels + augmented_labels
|
|
logger.info(f"증강 후 클래스별 샘플: {Counter(labels)}")
|
|
# ────────────────────────────────────────────────
|
|
|
|
# # ── 클래스 언더샘플링 ─────────────────────────
|
|
# MAX_SAMPLES_PER_CLASS = 20 # B타입 8개의 2~3배 수준으로 제한
|
|
#
|
|
# balanced_data = []
|
|
# class_counts = Counter(labels)
|
|
#
|
|
# for label in set(labels):
|
|
# class_texts = [t for t, l in zip(texts, labels) if l == label]
|
|
# # MAX_SAMPLES_PER_CLASS 초과 시 랜덤 샘플링
|
|
# if len(class_texts) > MAX_SAMPLES_PER_CLASS:
|
|
# class_texts = random.sample(class_texts, MAX_SAMPLES_PER_CLASS)
|
|
# balanced_data.extend([(t, label) for t in class_texts])
|
|
#
|
|
# texts, labels = zip(*balanced_data)
|
|
# texts, labels = list(texts), list(labels)
|
|
# logger.info(f"언더샘플링 후: {Counter(labels)}")
|
|
|
|
# # ── 핵심 키워드 가중치 부여 (특정 클래스만) ──────────────────────
|
|
# KEYWORD_BOOST_BY_LABEL = {
|
|
# "AMKOR TECHNOLOGY TAIWAN_A": { # A타입에만
|
|
# "Process Fee": 5,
|
|
# "Total Process Fee": 5,
|
|
# "Consigned Value": 5,
|
|
# "Total Process Value": 5,
|
|
# },
|
|
# "AMKOR TECHNOLOGY TAIWAN_B": { # B타입에만
|
|
# "PACKING LIST SUMMARY": 5,
|
|
# "FOC Y": 3,
|
|
# },
|
|
# }
|
|
#
|
|
# def boost_keywords(text: str, label: str) -> str:
|
|
# boost_map = KEYWORD_BOOST_BY_LABEL.get(label, {}) # 해당 label 없으면 빈 dict
|
|
# boosted = text
|
|
# for keyword, repeat in boost_map.items():
|
|
# if keyword.lower() in text.lower():
|
|
# boosted += f" {keyword}" * repeat
|
|
# return boosted
|
|
#
|
|
# texts = [boost_keywords(t, l) for t, l in zip(texts, labels)]
|
|
# logger.info("핵심 키워드 가중치 부여 완료")
|
|
# # ────────────────────────────────────────────────
|
|
|
|
# # ── 샘플 부족 클래스 필터링 ──────────────────────
|
|
# counts = Counter(labels)
|
|
# MIN_SAMPLES = 5
|
|
# before = len(texts)
|
|
# filtered = [(t, l) for t, l in zip(texts, labels) if counts[l] >= MIN_SAMPLES]
|
|
# if not filtered:
|
|
# raise ValueError("모든 클래스가 샘플 부족으로 제외되었습니다.")
|
|
# texts, labels = map(list, zip(*filtered))
|
|
# excluded = {k: v for k, v in counts.items() if v < MIN_SAMPLES}
|
|
# if excluded:
|
|
# logger.warning(f"샘플 부족 제외: {excluded}")
|
|
# logger.info(f"필터링: {before}개 → {len(texts)}개")
|
|
# # ────────────────────────────────────────────────
|
|
#
|
|
# counts = Counter(labels)
|
|
# min_count = min(counts.values())
|
|
# cv = 5
|
|
# stratify = labels if min_count >= cv else None
|
|
|
|
MIN_SAMPLES = 5
|
|
|
|
# ── 필터링 ───────────────────────────────────────
|
|
counts = Counter(labels)
|
|
before = len(texts)
|
|
filtered = [(t, l) for t, l in zip(texts, labels) if counts[l] >= MIN_SAMPLES]
|
|
if not filtered:
|
|
raise ValueError("모든 클래스가 샘플 부족으로 제외되었습니다.")
|
|
texts, labels = map(list, zip(*filtered))
|
|
excluded = {k: v for k, v in counts.items() if v < MIN_SAMPLES}
|
|
if excluded:
|
|
logger.warning(f"샘플 부족 제외: {excluded}")
|
|
logger.info(f"필터링: {before}개 → {len(texts)}개")
|
|
|
|
# ── MIN_SAMPLES 기준으로 cv/stratify 자동 결정 ──
|
|
counts = Counter(labels)
|
|
min_count = min(counts.values())
|
|
cv = max(2, min(5, int(MIN_SAMPLES * 0.8))) # MIN_SAMPLES의 80% (train 비율)
|
|
stratify = labels if min_count >= cv else None
|
|
|
|
logger.info(f"MIN_SAMPLES={MIN_SAMPLES} → cv={cv}, stratify={'적용' if stratify else '비적용'}")
|
|
|
|
_status["message"] = f"{len(texts)}개 샘플 / {len(counts)}개 클래스 학습 중..."
|
|
_status["samples"] = len(texts)
|
|
_status["classes"] = len(counts)
|
|
logger.info(_status["message"])
|
|
|
|
X_train, X_test, y_train, y_test = train_test_split(
|
|
texts, labels,
|
|
test_size=0.2,
|
|
stratify=stratify,
|
|
random_state=42,
|
|
)
|
|
|
|
# train_counts = Counter(y_train)
|
|
# min_train_count = min(train_counts.values())
|
|
# cv = max(2, min(5, min_train_count))
|
|
|
|
pipeline = Pipeline([
|
|
("tfidf", TfidfVectorizer(max_features=50000, ngram_range=(1, 2))),
|
|
("clf", CalibratedClassifierCV(LinearSVC(max_iter=2000, class_weight="balanced"), cv=cv)),
|
|
])
|
|
pipeline.fit(X_train, y_train)
|
|
|
|
report = classification_report(y_test, pipeline.predict(X_test), zero_division=0)
|
|
logger.info(f"\n{report}")
|
|
|
|
classifier_model_store.set_model(pipeline)
|
|
|
|
_status = {
|
|
"status": "done",
|
|
"message": f"학습 완료 | 샘플={len(texts)}, 클래스={len(counts)}",
|
|
"samples": len(texts),
|
|
"classes": len(counts),
|
|
"report": report, # 필요 시 별도 API로 노출
|
|
}
|
|
|
|
logger.info("classifier 학습 완료")
|
|
|
|
except Exception as e:
|
|
_status = {"status": "error", "message": str(e)}
|
|
logger.exception(f"classifier 학습 실패: {e}") |