Analyse 2: Keyword in Context (KWIC)#
0. Importe und Daten-Upload#
import pandas as pd
import re
from pathlib import Path
## for interactivity in jupyter books
from bokeh.io import output_notebook, show
from bokeh.layouts import column
from bokeh.models import CustomJS, TextInput, Div
# Ensure Bokeh output is displayed in the notebook
output_notebook()
conllfiles = Path(r"../data/csv")
corpus_annotations = {}
for file in conllfiles.iterdir():
if file.suffix == '.csv':
#path = os.path.join(conllfiles, filename)
data = pd.read_csv(file)
corpus_annotations[file.name] = data
corpus_metadata = pd.read_csv(Path('../data/metadata/QUADRIGA_FS-Text-01_Data01_Corpus-Table.csv'), sep=';')
corpus_metadata = corpus_metadata.set_index('DC.identifier')
1. KWIC-Suche#
Show code cell content
class ContextViewer:
def __init__(self, corpus_annotated, corpus_metadata):
self.prepare_index_dataframe_for_search(corpus_annotated, corpus_metadata)
def prepare_index_dataframe_for_search(self, corpus_annotated, corpus_metadata):
for filename, annotated_text in corpus_annotated.items():
txtname = filename.replace('.csv', '')
if txtname in corpus_metadata.index:
year, month, day = self.get_date_fname(txtname, corpus_metadata)
annotated_text['month'] = month
annotated_text['filename'] = filename
self.full_df = pd.concat(corpus_annotated.values())
self.full_df = self.full_df.reset_index()
print(f'Searching in a corpus of {self.full_df.shape[0]} word occurences')
def get_date_fname(self, txtname, corpus_metadata):
date = corpus_metadata.loc[txtname, 'DC.date']
date = str(date)
year = date[:4]
month = date[:7]
day = date
return year, month, day
def get_context_words(self, search_terms, n_words):
#search_terms = input('Insert a word to search, split by comma if more than one: ')
if len(search_terms) == 0:
search_terms = 'Grippe, Krankheit'
search_terms = search_terms.split(',')
search_terms = [x.strip() for x in search_terms]
indices = self.full_df.query(f'Lemma.isin({search_terms})').index
#print(indices)
left_contexts = []
this_words = []
right_contexts = []
months = []
for indice in indices:
left = self.full_df.iloc[indice-10:indice-1, ]["Token"]
leftс = left[~left.str.contains('\n')]
right = self.full_df.iloc[indice+1:indice+10, ]["Token"]
rightс = right[~right.str.contains('\n')]
left_contexts.append(' '.join(leftс))
right_contexts.append(' '.join(rightс))
this_words.append(self.full_df.iloc[indice, ]["Token"])
months.append(self.full_df.iloc[indice, ]["month"])
newdf = pd.DataFrame()
newdf['left_context'] = left_contexts
newdf['word'] = this_words
newdf['right_context'] = right_contexts
newdf['month'] = months
return newdf
## currently unused functionality:
def get_context_sents(self, n_sentences):
search_lemma = input('Insert a word to search: ')
if len(search_lemma) == 0:
search_lemma = 'Grippe'
indices = self.full_df.query(f'Lemma=="{search_lemma}"').index
#print(indices)
left_contexts = []
this_sentences = []
right_contexts = []
months = []
for indice in indices:
#print(indice)
current_filename = self.full_df.iloc[indice, ]["filename"]
current_sentence_id = self.full_df.iloc[indice, ]["Sentence_idx"]
left_context = self.get_sents(direction=-1,
current_filename=current_filename,
current_sentence_id=current_sentence_id,
n_sentences=n_sentences)
left_contexts.append(left_context)
right_context = self.get_sents(direction=1,
current_filename=current_filename,
current_sentence_id=current_sentence_id,
n_sentences=n_sentences)
right_contexts.append(right_context)
this_sentence = self.get_sents(direction=0,
current_filename=current_filename,
current_sentence_id=current_sentence_id,
n_sentences=1)
this_sentences.append(this_sentence)
#this_words.append(self.full_df.iloc[indice, ]["Token"])
months.append(self.full_df.iloc[indice, ]["month"])
newdf = pd.DataFrame()
newdf['left_sentences'] = left_contexts
newdf['this_sentence'] = this_sentences
newdf['right_sentences'] = right_contexts
newdf['month'] = months
return newdf #.sort_values(by='month')
def get_sents(self, direction, current_filename, current_sentence_id, n_sentences):
sentences = []
for n in range(1,n_sentences+1):
sentence_id = current_sentence_id + (n * direction)
this_sentence = self.create_sentence(current_filename, sentence_id)
sentences.append(this_sentence)
#print(' '.join(sentences))
return ' '.join(sentences)
def create_sentence(self, current_filename, sentence_id):
words = self.full_df.query(f'filename=="{current_filename}" and Sentence_idx=={sentence_id}')['Token']
sentence = ' '.join(words)
#print(sentence)
return sentence
search_terms = TextInput(value='Grippe, Krankheit',
title="Geben Sie die zu suchenden Wörter ein und trennen Sie sie durch Kommas, wenn es mehrere sind:") #input('Insert words to search, split by comma if more than one: ')
search_terms_str = search_terms.value.strip()
# JavaScript callback to update the in Jupyter Book
rewrite_var_after_input = CustomJS(args=dict(text_input=search_terms), code="""
var word = text_input.value.trim();
console.log('Input value:', word);
function sendToPython(){
var kernel = IPython.notebook.kernel;
kernel.execute("search_terms_str = '" + word + "'");
}
sendToPython();
""")
search_terms.js_on_change('value', rewrite_var_after_input)
# Layout and display
layout = column(search_terms)
show(layout)
kwic = ContextViewer(corpus_annotations, corpus_metadata)
Searching in a corpus of 33192061 word occurences
kwic.get_context_words(search_terms_str, n_words=5)
| left_context | word | right_context | month | |
|---|---|---|---|---|
| 0 | nach Herzenslust ausfoben . Onkel Gustav , der... | Krankheit | die männliche Aufsicht bei uns führte , hatte ... | 1919-07 |
| 1 | Alto Ch NIH 8 » 86 kurzer , | Krankheit | " it . am 4 , d. Mis , | 1919-06 |
| 2 | ENEN nN K208 n -- A aathuneg | I | Krankheit | | tal : jrenve : tau ! Vjerdeaeschixre , | 1918-08 |
| 3 | dia Rn Näheres wird noch bekannt » . | Krankheit | ist mir in der Nacht 906 , tupe 14 | 1918-12 |
| 4 | Deutschland , geb. Vasch verschied rsehied hat... | Krankheit | au Zug , go Deutschland , ue Halle | 1918-12 |
| ... | ... | ... | ... | ... |
| 1257 | | I der gestrigen Gihung des DBandtarates | an | Grippe | , einer stub . oder an einem bestimmten Ort | 1918-10 |
| 1258 | - im freien ugkampf , ist nach einer | Krankheit | in jew York gestorben . * beiden größten | 1918-02 |
| 1259 | Gein Yin 28. April verstarb nach kurzer , | Krankheit | mein Uebes Found , Harz . Josef Cohen | 1918-04 |
| 1260 | der Kassenbote NETE cn . den Folgen | Grippe | Herr - Herr Paul Nagel . Galt | 1919-01 |
| 1261 | traten die ersten Anzeichen yon Skor- but auf , | Krankheit | , die dur < ' schlechtes Trinkwasser und Mängel | 1918-02 |
1262 rows × 4 columns