NLP: Financial News Classification

import json
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.neural_network import MLPClassifier
from sklearn.decomposition import TruncatedSVD
from sklearn.metrics import classification_report, confusion_matrix

# === STEP 1: Load Real Reuters-21578 Dataset from API ===
# Fetch authentic financial news articles from Reuters-21578 corpus
# Categories: earn (earnings), acq (acquisitions), trade (tariffs), crude (oil)
from js import fetch
import json as json_module

np.random.seed(42)

# Fetch real Reuters articles from API
response = await fetch('/api/ReutersData/articles')
text = await response.text()
samples = json_module.loads(text)

# Create dataset with labels
texts = []
labels = []
for category, docs in samples.items():
    texts.extend(docs)
    labels.extend([category] * len(docs))

# Split into train (70%) and test (30%)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(
    texts, labels, test_size=0.3, random_state=42, stratify=labels
)

# === STEP 2: TF-IDF Vectorization ===
# Convert text to numerical features using Term Frequency-Inverse Document Frequency
vectorizer = TfidfVectorizer(max_features=500, stop_words='english')
X_train_tfidf = vectorizer.fit_transform(X_train)
X_test_tfidf = vectorizer.transform(X_test)

# === STEP 3: Baseline Model - Multinomial Naive Bayes ===
# Statistical classifier assuming feature independence (Bag of Words)
nb_model = MultinomialNB()
nb_model.fit(X_train_tfidf, y_train)
nb_predictions = nb_model.predict(X_test_tfidf)

# Calculate metrics
nb_report = classification_report(y_test, nb_predictions, output_dict=True, zero_division=0)
nb_cm = confusion_matrix(y_test, nb_predictions, labels=['acq', 'crude', 'earn', 'trade'])

# === STEP 4: LSA Dimensionality Reduction ===
# Compress high-dimensional TF-IDF vectors to capture latent semantic structure
lsa = TruncatedSVD(n_components=20, random_state=42)
X_train_lsa = lsa.fit_transform(X_train_tfidf)
X_test_lsa = lsa.transform(X_test_tfidf)

# Calculate explained variance
explained_variance = lsa.explained_variance_ratio_[:10]  # Top 10 components

# === STEP 5: Neural Network Model - MLP on LSA Embeddings ===
# Multi-Layer Perceptron for non-linear classification
mlp_model = MLPClassifier(hidden_layer_sizes=(50, 25), max_iter=500, random_state=42)
mlp_model.fit(X_train_lsa, y_train)
mlp_predictions = mlp_model.predict(X_test_lsa)

# Calculate metrics
mlp_report = classification_report(y_test, mlp_predictions, output_dict=True, zero_division=0)
mlp_cm = confusion_matrix(y_test, mlp_predictions, labels=['acq', 'crude', 'earn', 'trade'])

# === STEP 6: Prepare Output ===
results = {
    'dataset_stats': {
        'total_docs': len(texts),
        'train_docs': len(X_train),
        'test_docs': len(X_test),
        'categories': list(samples.keys()),
        'vocab_size': len(vectorizer.vocabulary_)
    },
    'naive_bayes': {
        'accuracy': float(nb_report['accuracy']),
        'precision': float(nb_report['weighted avg']['precision']),
        'recall': float(nb_report['weighted avg']['recall']),
        'f1_score': float(nb_report['weighted avg']['f1-score']),
        'per_class': {
            cat: {
                'precision': float(nb_report[cat]['precision']),
                'recall': float(nb_report[cat]['recall']),
                'f1': float(nb_report[cat]['f1-score'])
            } for cat in ['acq', 'crude', 'earn', 'trade'] if cat in nb_report
        },
        'confusion_matrix': nb_cm.tolist()
    },
    'mlp_lsa': {
        'accuracy': float(mlp_report['accuracy']),
        'precision': float(mlp_report['weighted avg']['precision']),
        'recall': float(mlp_report['weighted avg']['recall']),
        'f1_score': float(mlp_report['weighted avg']['f1-score']),
        'per_class': {
            cat: {
                'precision': float(mlp_report[cat]['precision']),
                'recall': float(mlp_report[cat]['recall']),
                'f1': float(mlp_report[cat]['f1-score'])
            } for cat in ['acq', 'crude', 'earn', 'trade'] if cat in mlp_report
        },
        'confusion_matrix': mlp_cm.tolist(),
        'lsa_components': 20,
        'explained_variance': [float(v) for v in explained_variance]
    }
}

print(json.dumps(results))