FL-record-linkage/test_only_data_generator.py at main · mikez3/FL-record-linkage · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
# Import libs
import pandas as pd
import numpy as np
import cupy as cp
from nameparser import HumanName
from rapidfuzz.distance import Levenshtein
from scipy.spatial.distance import cdist
import cudf
from tqdm import tqdm
import dask.array as da
# from numba import njit

metric = 'cosine'
# euclidean,cityblock,seuclidean,sqeuclidean,cosine,correlation,hamming,jaccard,jensenshannon,chebyshev,canberra,braycurtis,-NOT-mahalanobis, custom_distance
test_rows = 5000
refrows = 200
chunk_size = 5000

if isinstance(metric, str):
    filename = '/Data/test samples/test_'+metric+'_n'+str(test_rows)+'_ref'+str(refrows)+'.csv'
elif callable(metric):
    filename = '/Data/test samples/test_'+ str(metric.__name__)+'_n'+str(test_rows)+'_ref'+str(refrows)+'.csv'


dfA_test = pd.read_csv('Data/BIASA_200000.csv', delimiter="|", header=None, nrows=test_rows).drop([3,4,5,6], axis=1).rename({0:'LastName', 1:'FirstName', 2:'MiddleName',7:'id'}, axis='columns')
dfB_test = pd.read_csv('Data/BIASB_200000.csv', delimiter="|", header=None, nrows=test_rows).drop([3,4,5,6], axis=1).rename({0:'LastName', 1:'FirstName', 2:'MiddleName',7:'id'}, axis='columns')

dfA_test = dfA_test[['id', 'FirstName', 'LastName', 'MiddleName']].astype(str)
dfB_test = dfB_test[['id', 'FirstName', 'LastName', 'MiddleName']].astype(str)

# for spanish names
# dfA = pd.read_csv('Data/authors3_A.csv', header=None, encoding='latin-1', nrows=nrows).rename({0:'id', 1:'FirstName', 2:'MiddleName',3:'LastName'}, axis='columns')
# dfB = pd.read_csv('Data/authors3_B.csv', header=None, encoding='latin-1', nrows=nrows).rename({0:'id', 1:'FirstName', 2:'MiddleName',3:'LastName'}, axis='columns')
# dfA = dfA[['id', 'FirstName', 'LastName', 'MiddleName']].astype(str)
# dfB = dfB[['id', 'FirstName', 'LastName', 'MiddleName']].astype(str)


ref = pd.read_csv('reference_set.csv', names=['name'], nrows=refrows)
# Extract the first and last names from the parsed names
ref['ParsedName'] = ref['name'].str.replace('_', ' ').apply(lambda x: HumanName(x))
ref["FirstName"] = ref["ParsedName"].apply(lambda x: x.first)
ref["LastName"] = ref["ParsedName"].apply(lambda x: x.last)
ref = ref.drop(['ParsedName', 'name'], axis=1)

# A-B
# Α-ref
edit_distances_AtoRef_names = []
edit_distances_AtoRef_lastnames = []
edit_distances_AtoRef_middlenames_first = []
edit_distances_AtoRef_middlenames_last = []
# edit_distances_AtoRef_names_lastnames = []
# edit_distances_AtoRef_lastnames_names = []

for nameA in dfA_test.itertuples(index=False):
    # FirstName A - FirstName ref
    edit_distances_AtoRef_names.append([Levenshtein.distance(nameA.FirstName.upper(), nameRef) for nameRef in ref['FirstName']])
    # LastName A - LastName ref
    edit_distances_AtoRef_lastnames.append([Levenshtein.distance(nameA.LastName.upper(), nameRef) for nameRef in ref['LastName']])
    # MiddleName A - Firstname ref
    edit_distances_AtoRef_middlenames_first.append([Levenshtein.distance(nameA.MiddleName.upper(), nameRef) for nameRef in ref['FirstName']])
    # MiddleName A - Lastname ref
    edit_distances_AtoRef_middlenames_last.append([Levenshtein.distance(nameA.MiddleName.upper(), nameRef) for nameRef in ref['LastName']])
    # # FirstName A - LastName ref
    # edit_distances_AtoRef_names_lastnames.append([Levenshtein.distance(nameA.FirstName.upper(), nameRef) for nameRef in ref['LastName']])
    # # LastName A - FirstName ref
    # edit_distances_AtoRef_lastnames_names.append([Levenshtein.distance(nameA.LastName.upper(), nameRef) for nameRef in ref['FirstName']])

edit_distances_AtoRef_names = np.array(edit_distances_AtoRef_names)
edit_distances_AtoRef_lastnames = np.array(edit_distances_AtoRef_lastnames)
edit_distances_AtoRef_middlenames_first = np.array(edit_distances_AtoRef_middlenames_first)
edit_distances_AtoRef_middlenames_last = np.array(edit_distances_AtoRef_middlenames_last)
# edit_distances_AtoRef_names_lastnames = np.array(edit_distances_AtoRef_names_lastnames)
# edit_distances_AtoRef_lastnames_names = np.array(edit_distances_AtoRef_lastnames_names)

# B-ref
edit_distances_BtoRef_names = []
edit_distances_BtoRef_lastnames = []
edit_distances_BtoRef_middlenames_first = []
edit_distances_BtoRef_middlenames_last = []
# edit_distances_BtoRef_names_lastnames = []
# edit_distances_BtoRef_lastnames_names = []

for nameB in dfB_test.itertuples(index=False):
    # FirstName B - FirstName ref
    edit_distances_BtoRef_names.append([Levenshtein.distance(nameB.FirstName.upper(), nameRef) for nameRef in ref['FirstName']])
    # LastName B - LastName ref
    edit_distances_BtoRef_lastnames.append([Levenshtein.distance(nameB.LastName.upper(), nameRef) for nameRef in ref['LastName']])
    # MiddleName B - FirstName ref
    edit_distances_BtoRef_middlenames_first.append([Levenshtein.distance(nameB.MiddleName.upper(), nameRef) for nameRef in ref['FirstName']])
    # MiddleName B - Lastname ref
    edit_distances_BtoRef_middlenames_last.append([Levenshtein.distance(nameB.MiddleName.upper(), nameRef) for nameRef in ref['LastName']])
    # # FirstName B - LastName ref
    # edit_distances_BtoRef_names_lastnames.append([Levenshtein.distance(nameB.FirstName.upper(), nameRef) for nameRef in ref['LastName']])
    # # # LastName B - FirstName ref
    # edit_distances_BtoRef_lastnames_names.append([Levenshtein.distance(nameB.LastName.upper(), nameRef) for nameRef in ref['FirstName']])

edit_distances_BtoRef_names = np.array(edit_distances_BtoRef_names)
edit_distances_BtoRef_lastnames = np.array(edit_distances_BtoRef_lastnames)
edit_distances_BtoRef_middlenames_first = np.array(edit_distances_BtoRef_middlenames_first)
edit_distances_BtoRef_middlenames_last = np.array(edit_distances_BtoRef_middlenames_last)
# edit_distances_BtoRef_names_lastnames = np.array(edit_distances_BtoRef_names_lastnames)
# edit_distances_BtoRef_lastnames_names = np.array(edit_distances_BtoRef_lastnames_names)
# %%
num_chunks = len(edit_distances_AtoRef_names) // chunk_size + 1
with open(filename, 'w') as f:
    for i in tqdm(range (num_chunks)):
            start_i = i * chunk_size
            end_i = (i + 1) * chunk_size
            for j in tqdm(range (num_chunks)):
                start_j = j * chunk_size
                end_j = (j + 1) * chunk_size
                # Convert arrays to Dask arrays
                edit_distances_AtoRef_names_dask = da.from_array(edit_distances_AtoRef_names[start_i:end_i])
                edit_distances_BtoRef_names_dask = da.from_array(edit_distances_BtoRef_names[start_j:end_j])
                edit_distances_AtoRef_lastnames_dask = da.from_array(edit_distances_AtoRef_lastnames[start_i:end_i])
                edit_distances_BtoRef_lastnames_dask = da.from_array(edit_distances_BtoRef_lastnames[start_j:end_j])
                edit_distances_AtoRef_middlenames_first_dask = da.from_array(edit_distances_AtoRef_middlenames_first[start_i:end_i])
                edit_distances_BtoRef_middlenames_first_dask = da.from_array(edit_distances_BtoRef_middlenames_first[start_j:end_j])
                edit_distances_AtoRef_middlenames_last_dask = da.from_array(edit_distances_AtoRef_middlenames_last[start_i:end_i])
                edit_distances_BtoRef_middlenames_last_dask = da.from_array(edit_distances_BtoRef_middlenames_last[start_j:end_j])

                distances_names = da.map_blocks(lambda x, y: cdist(x, y, metric), edit_distances_AtoRef_names_dask, edit_distances_BtoRef_names_dask).compute()
                distances_names = cp.asarray(distances_names)

                distances_lastnames = da.map_blocks(lambda x, y: cdist(x, y, metric), edit_distances_AtoRef_lastnames_dask, edit_distances_BtoRef_lastnames_dask).compute()
                distances_lastnames = cp.asarray(distances_lastnames)

                distances_middlenames_first = da.map_blocks(lambda x, y: cdist(x, y, metric), edit_distances_AtoRef_middlenames_first_dask, edit_distances_BtoRef_middlenames_first_dask).compute()
                distances_middlenames_first = cp.asarray(distances_middlenames_first)

                distances_middlenames_last = da.map_blocks(lambda x, y: cdist(x, y, metric), edit_distances_AtoRef_middlenames_last_dask, edit_distances_BtoRef_middlenames_last_dask).compute()
                distances_middlenames_last = cp.asarray(distances_middlenames_last)

                # _-------------------------------------

                # For cosine, correlation:
                # GPU accelerated
                # distances_names = (cucdist((edit_distances_AtoRef_names[start_i:end_i]), (edit_distances_BtoRef_names[start_j:end_j]), metric=metric))
                # distances_lastnames = (cucdist((edit_distances_AtoRef_lastnames[start_i:end_i]), (edit_distances_BtoRef_lastnames[start_j:end_j]), metric=metric))
                # distances_middlenames_first = (cucdist((edit_distances_AtoRef_middlenames_first[start_i:end_i]), (edit_distances_BtoRef_middlenames_first[start_j:end_j]), metric=metric))
                # distances_middlenames_last = (cucdist((edit_distances_AtoRef_middlenames_last[start_i:end_i]), (edit_distances_BtoRef_middlenames_last[start_j:end_j]), metric=metric))

                # no GPU
                # distances_names = cdist(edit_distances_AtoRef_names[start_i:end_i], edit_distances_BtoRef_names[start_j:end_j], metric)
                # distances_lastnames = cdist(edit_distances_AtoRef_lastnames[start_i:end_i], edit_distances_BtoRef_lastnames[start_j:end_j], metric)
                # distances_middlenames_first = cdist(edit_distances_AtoRef_middlenames_first[start_i:end_i], edit_distances_BtoRef_middlenames_first[start_j:end_j], metric)
                # distances_middlenames_last = cdist(edit_distances_AtoRef_middlenames_last[start_i:end_i], edit_distances_BtoRef_middlenames_last[start_j:end_j], metric)

                # distances_names = 1 - fastdist.matrix_to_matrix_distance(edit_distances_AtoRef_names[start_i:end_i], edit_distances_BtoRef_names[start_j:end_j], fastdist.cosine, metric)
                # distances_lastnames = 1 - fastdist.matrix_to_matrix_distance(edit_distances_AtoRef_lastnames[start_i:end_i], edit_distances_BtoRef_lastnames[start_j:end_j], fastdist.cosine, metric)
                # distances_middlenames_first = 1 - fastdist.matrix_to_matrix_distance(edit_distances_AtoRef_middlenames_first[start_i:end_i], edit_distances_BtoRef_middlenames_first[start_j:end_j], fastdist.cosine, metric)
                # distances_middlenames_last = 1 - fastdist.matrix_to_matrix_distance(edit_distances_AtoRef_middlenames_last[start_i:end_i], edit_distances_BtoRef_middlenames_last[start_j:end_j], fastdist.cosine, metric)

                # distances_first_last = np.array(cdist(edit_distances_AtoRef_names_lastnames, edit_distances_BtoRef_names_lastnames, metric=metric))
                # distances_last_first = np.array(cdist(edit_distances_AtoRef_lastnames_names, edit_distances_BtoRef_lastnames_names, metric=metric))
                # distances = np.column_stack(((distances_names).ravel(), (distances_lastnames).ravel(), (distances_middlenames_first).ravel()))

                distances = cp.column_stack((cp.asarray(distances_names).ravel(), cp.asarray(distances_lastnames).ravel(), cp.asarray(distances_middlenames_first).ravel(), cp.asarray(distances_middlenames_last).ravel()))
                comparisons_with_labels = cudf.DataFrame((dfA_test['id'].values[start_i:end_i, None] == dfB_test['id'].values[start_j:end_j]).astype(int).ravel(), columns=['label'])
                distances_with_labels = cp.column_stack([cp.asarray(comparisons_with_labels['label']), cp.asarray(distances)])

                dataAB = cudf.DataFrame(distances_with_labels)
                # Save the data
                dataAB.to_csv(f, index=False, header=None)