import pdfquery from lxml import etree from .formatter import formatter from .writer import writer class reader: """Reads table data from KPN mobile incoices""" def __init__(self, writer, table_names, first_column_names): self.writer = writer self.first_column_names = first_column_names self.requested_tables = table_names def load(self, file): self.file = file print('Loading %s' % str(self.file.split('\\')[-1:])) self.pdf = pdfquery.PDFQuery(self.file) page_count = len(self.pdf._pages) #self.pdf.load(list(range(2,page_count))) self.pdf.load() #self.write_tree_to_xml() def write_tree_to_xml(self): with open('xmltree.xml','wb') as f: f.write(etree.tostring(self.pdf.tree, pretty_print=True)) @staticmethod def get_coordinates(query): assert 'x0' in query.attrib and 'y0' in query.attrib, 'querry doesn\'t contain coordinates' return { 'x' : float(query.attrib['x0']), 'y' : float(query.attrib['y0'])} @staticmethod def get_pageid(element) -> int: pageid = next(element.iterancestors('LTPage')).attrib['pageid'] if pageid == '': raise ValueError('element doesn\'t have a ancestor with pageid') return int(pageid) def text_line_in_box_in_page(self, x0, y0, x1, y1, pageid, textbox = False): lines = self.pdf.pq('%s:in_bbox("%s, %s, %s, %s")' % ('LTTextBoxHorizontal' if textbox else 'LTTextLineHorizontal', x0, y0, x1, y1)) output = [line for line in lines if reader.get_pageid(line) == pageid] return output def fix_boxvalue(self, line_results, coords, pageid_number): line_box_results = self.text_line_in_box_in_page(coords['x']+1, coords['y']-2, coords['x']+300, coords['y'] + 8, pageid_number, True) line_box_len = len(line_box_results) line_box_index = 0 for i in range(0, len(line_results)): if line_box_index >= line_box_len: return line_results if line_results[i].text == '' and line_box_results[line_box_index].text != '': line_results[i] = line_box_results[line_box_index] line_box_index += 1 return line_results def read_table_line(self, first_element, header = False) -> str: formatter.remove_prefix(first_element) line_elements = [] line_elements.append(first_element) coords = reader.get_coordinates(first_element) pageid_number = reader.get_pageid(first_element) line_results = self.text_line_in_box_in_page(coords['x']+1, coords['y']-2, coords['x']+300, coords['y'] + 8, pageid_number) if '' in [x.text for x in line_results]: line_results = self.fix_boxvalue(line_results , coords, pageid_number) line_elements += line_results line_elements = sorted(line_elements, key=lambda x: float(x.attrib['x0'])) line = [element.text.strip() for element in line_elements] if header: self.line_buffer.append(formatter.concat_line(line)) else: line = formatter.date_time_merger(line, self.invoice_date) return formatter.concat_line(line) def read_table_part(self, header): coords = reader.get_coordinates(header) last_label = None while True: label_number = self.text_line_in_box_in_page(coords['x'], coords['y']-10, coords['x']+80, coords['y'], reader.get_pageid(header)) if len(label_number) == 0: break line = self.read_table_line(label_number[0]) print(line[:-1]) self.line_buffer.append(line) coords = reader.get_coordinates(label_number[0]) last_label = label_number[0] return last_label def check_for_continuation(self, last_label, header_text): pageid_last_label = reader.get_pageid(last_label) coords_last_label = reader.get_coordinates(last_label) label_totaal = self.text_line_in_box_in_page(coords_last_label['x'] - 1, coords_last_label['y']-15, coords_last_label['x']+165, coords_last_label['y'], pageid_last_label, True) if len(label_totaal) != 0: if label_totaal[0].text[:6] == 'Totaal' : return None for label_vervolg in self.pdf.pq('LTTextLineHorizontal:contains("vervolg")'): coords = reader.get_coordinates(label_vervolg) if not 700 < coords['y'] < 715: continue label_vervolg_page = reader.get_pageid(label_vervolg) if coords_last_label['x'] < 300: if label_vervolg_page != pageid_last_label: continue if coords['x'] < 300: continue else: if label_vervolg_page != pageid_last_label + 1: continue if coords['x'] > 300: continue label_header2 = self.text_line_in_box_in_page(coords['x'], coords['y']-12, coords['x']+150, coords['y'], label_vervolg_page)[0] if label_header2.text.strip() != header_text: continue return label_header2 return None def read_table(self, label_header): label_start = label_header while True: last_label = self.read_table_part(label_start) if last_label is None: break label_start = self.check_for_continuation(last_label, label_header.text.strip()) if label_start is None: break # No label matched return def read_table_data(self, table_name): for label_table_name in self.pdf.pq('LTTextLineHorizontal:contains("%s")' % table_name): self.line_buffer = [] table_name_coords = reader.get_coordinates(label_table_name) label_header = self.text_line_in_box_in_page(table_name_coords['x'], table_name_coords['y']-10, table_name_coords['x']+75, table_name_coords['y'], reader.get_pageid(label_table_name)) if len(label_header) == 0 : continue if label_header[0].text.strip() in self.first_column_names: self.read_table_line(label_header[0], True) self.read_table(label_header[0]) self.writer.write_buffer_to_file(self.line_buffer, table_name) break return def get_invoice_date(self): date = self.text_line_in_box_in_page(440, 597, 519, 608, 1)[0] return date.text.strip() def read_pdf(self): print('Reading %s' % str(self.file.split('\\')[-1:])) self.invoice_date = self.get_invoice_date() for table_name in self.requested_tables: self.read_table_data(table_name) # in_bbox("x0,y0,x1,y1"): Matches only elements that fit entirely within the given bbox. # overlaps_bbox("x0,y0,x1,y1"): Matches any elements that overlap the given bbox.