特征提取 #

from sklearn.feature_extraction.text import CountVectorizer

corpus = [
    'This is the first document.',
    'This document is the second document.',
    'And this is the third one.',
    'Is this the first document?'
]

vectorizer = CountVectorizer()
X = vectorizer.fit_transform(corpus)

print(f"词汇表: {vectorizer.get_feature_names_out()}")
print(f"特征矩阵形状: {X.shape}")
print(f"特征矩阵:\n{X.toarray()}")

vectorizer = CountVectorizer(
    max_features=1000,
    min_df=2,
    max_df=0.95,
    ngram_range=(1, 2),
    stop_words='english'
)

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer()
X = tfidf.fit_transform(corpus)

print(f"TF-IDF 矩阵:\n{X.toarray()}")
print(f"IDF 值: {tfidf.idf_}")

from sklearn.feature_extraction.text import TfidfTransformer

count_vec = CountVectorizer()
X_counts = count_vec.fit_transform(corpus)

tfidf_transformer = TfidfTransformer()
X_tfidf = tfidf_transformer.fit_transform(X_counts)

import jieba

def chinese_tokenizer(text):
    return list(jieba.cut(text))

corpus_cn = [
    '我喜欢学习机器学习',
    '机器学习很有趣',
    '深度学习是机器学习的分支'
]

vectorizer = CountVectorizer(tokenizer=chinese_tokenizer)
X = vectorizer.fit_transform(corpus_cn)

from sklearn.feature_extraction.text import HashingVectorizer

hash_vec = HashingVectorizer(
    n_features=2**10,
    alternate_sign=False
)
X = hash_vec.transform(corpus)

print(f"特征矩阵形状: {X.shape}")

from sklearn.feature_extraction import DictVectorizer

measurements = [
    {'city': 'Dubai', 'temperature': 33.},
    {'city': 'London', 'temperature': 12.},
    {'city': 'San Francisco', 'temperature': 18.}
]

vec = DictVectorizer(sparse=False)
X = vec.fit_transform(measurements)

print(f"特征名: {vec.get_feature_names_out()}")
print(f"特征矩阵:\n{X}")

data = [
    {'gender': 'male', 'age': 25},
    {'gender': 'female', 'age': 30},
    {'gender': 'male', 'age': 35}
]

vec = DictVectorizer(sparse=False)
X = vec.fit_transform(data)

print(f"特征名: {vec.get_feature_names_out()}")

from sklearn.datasets import load_digits
import matplotlib.pyplot as plt

digits = load_digits()
X, y = digits.data, digits.target

print(f"图像形状: {digits.images[0].shape}")
print(f"特征向量长度: {X.shape[1]}")

plt.imshow(digits.images[0], cmap='gray')
plt.title(f'Label: {y[0]}')

from sklearn.feature_extraction.image import PatchExtractor
import numpy as np

image = np.random.rand(100, 100)

extractor = PatchExtractor(patch_size=(10, 10), max_patches=10)
patches = extractor.transform([image])

print(f"提取的 patches 形状: {patches.shape}")

from skimage.feature import hog
from skimage import exposure

def extract_hog_features(image):
    features, hog_image = hog(
        image,
        orientations=8,
        pixels_per_cell=(4, 4),
        cells_per_block=(1, 1),
        visualize=True
    )
    return features, hog_image

features, hog_image = extract_hog_features(digits.images[0])
print(f"HOG 特征长度: {len(features)}")

from sklearn.feature_extraction import FeatureHasher

data = [
    {'dog': 1, 'cat': 2, 'elephant': 4},
    {'dog': 2, 'run': 5}
]

hasher = FeatureHasher(n_features=10)
X = hasher.transform(data)

print(f"哈希特征矩阵:\n{X.toarray()}")

hasher = FeatureHasher(n_features=2**10, input_type='string')
X = hasher.transform([['hello', 'world'], ['hello', 'sklearn']])

from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

texts = ['positive text', 'negative text', 'great news', 'bad news']
labels = [1, 0, 1, 0]

X_train, X_test, y_train, y_test = train_test_split(texts, labels, random_state=42)

pipe = Pipeline([
    ('tfidf', TfidfVectorizer()),
    ('clf', LogisticRegression())
])

pipe.fit(X_train, y_train)
print(f"准确率: {pipe.score(X_test, y_test):.4f}")

from sklearn.feature_selection import SelectKBest, chi2

pipe = Pipeline([
    ('tfidf', TfidfVectorizer(max_features=10000)),
    ('select', SelectKBest(chi2, k=1000)),
    ('clf', LogisticRegression())
])

vectorizer = CountVectorizer(ngram_range=(1, 3))
X = vectorizer.fit_transform(corpus)

print(f"1-3 gram 特征数: {X.shape[1]}")

char_vec = CountVectorizer(analyzer='char', ngram_range=(2, 4))
X = char_vec.fit_transform(corpus)

print(f"字符级特征数: {X.shape[1]}")

from nltk.stem import PorterStemmer
import nltk

stemmer = PorterStemmer()

def stem_tokenizer(text):
    return [stemmer.stem(word) for word in text.split()]

vectorizer = CountVectorizer(tokenizer=stem_tokenizer)

from nltk.stem import WordNetLemmatizer

lemmatizer = WordNetLemmatizer()

def lemma_tokenizer(text):
    return [lemmatizer.lemmatize(word) for word in text.split()]

from sklearn.datasets import fetch_20newsgroups
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import classification_report

categories = ['alt.atheism', 'soc.religion.christian', 'comp.graphics', 'sci.med']
newsgroups = fetch_20newsgroups(subset='train', categories=categories)

pipe = Pipeline([
    ('tfidf', TfidfVectorizer(stop_words='english')),
    ('clf', MultinomialNB())
])

pipe.fit(newsgroups.data, newsgroups.target)

newsgroups_test = fetch_20newsgroups(subset='test', categories=categories)
y_pred = pipe.predict(newsgroups_test.data)

print(classification_report(newsgroups_test.target, y_pred, 
                            target_names=newsgroups_test.target_names))

from sklearn.linear_model import LogisticRegression

reviews = [
    'This movie is great!',
    'Terrible experience',
    'I love this product',
    'Waste of money'
]
sentiments = [1, 0, 1, 0]

pipe = Pipeline([
    ('tfidf', TfidfVectorizer(ngram_range=(1, 2))),
    ('clf', LogisticRegression())
])

pipe.fit(reviews, sentiments)

from sklearn.model_selection import GridSearchCV

param_grid = {
    'tfidf__max_features': [1000, 5000, 10000],
    'tfidf__ngram_range': [(1, 1), (1, 2)],
    'tfidf__min_df': [1, 2, 5]
}

grid_search = GridSearchCV(pipe, param_grid, cv=5)

vectorizer = TfidfVectorizer(
    max_features=5000,
    dtype=np.float32
)

vectorizer = CountVectorizer(
    max_features=10000,
    n_jobs=-1
)