textcomplexity/pca.py at main · jegny100/textcomplexity · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
# -*- coding: utf-8 -*-
import pandas as pd
from pathlib import Path
from tqdm import tqdm
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

feature_folder = 'snippets_with_features'
df_standard = pd.concat([pd.read_csv(file)
                        for file in tqdm(Path(feature_folder).glob('*'))])

feature_folder = 'simple_snippets_with_features'
df_simple = pd.concat([pd.read_csv(file)
                      for file in tqdm(Path(feature_folder).glob('*'))])

df_standard.head()


df_combined = pd.concat([df_standard, df_simple], axis=0)
df_combined = df_combined[['NER', 'a1', 'a2', 'b1', 'b2', 'c1', 'unknown']]
# PCA auf der gesamten Datenmenge durchführen
pca = PCA(n_components=2)
pca_result = pca.fit_transform(df_combined)

# Erstellen eines DataFrames für die Ergebnisse von PCA
df_pca = pd.DataFrame(
    data={'PCA1': pca_result[:, 0], 'PCA2': pca_result[:, 1]})

# Scatterplot erstellen
plt.scatter(df_pca['PCA1'][:len(df_standard)], df_pca['PCA2']
            [:len(df_standard)], color='green', label='df_standard')
plt.scatter(df_pca['PCA1'][len(df_standard):], df_pca['PCA2']
            [len(df_standard):], color='red', label='df_simple')

# Achsentitel und Legende hinzufügen
plt.xlabel('PCA1')
plt.ylabel('PCA2')
plt.legend()
plt.title('Scatterplot nach PCA')

# Plot anzeigen
plt.show()

plt.hist2d(df_pca['PCA1'][:len(df_standard)], df_pca['PCA2']
           [:len(df_standard)], bins=(50, 50), cmap='Blues')
plt.colorbar()

plt.hist2d(df_pca['PCA1'][len(df_standard):], df_pca['PCA2']
           [len(df_standard):], bins=(50, 50), cmap='Blues')
plt.colorbar()

df_pca['PCA2'][len(df_standard):].describe()
df_pca['PCA2'][:len(df_standard)].describe()