Python 3 automated file batch processing (text, PDF; read, filter, export)

Here is a brief summary of the key features.

Read ini configuration file

1 import configparser
2
3 4 def getConfigInfo(_ini_nm): 5 6 # Open Ini File 7 config = configparser.ConfigParser() 8 if not config.read(os.path.join(os.getcwd(), _ini_nm + r'.ini')): 9 printLog('E', 'Read Ini file fail.') 10 11 while True: 12 sysCode = input(r'Please input the system code : (Press [Enter] to quit):').strip() 13 if 0 == len(sysCode.strip()): 14 exit() 15 16 # Init ConnectionSettings 17 if sysCode in config.sections(): 18 return dict(config[sysCode]) 19 else: 20 print('Ini info of System [%s] is blank.\n' % sysCode)

Acquisition of multi parameter input

1 def _main(): 2 3 path = '' 4 keyWord = '' 5 6 while True: 7 para = input(r'Please input the PDF directory and Key Word: (Press [Enter] to quit):').strip().split() 8 9 if 2 != len(para): continue 10 11 path = para[0] 12 keyWord = para[1] 13 14 if 0 == len(path.strip()): 15 exit() 16 17 if not os.path.exists(path): 18 print('input path is not a exists path.' + '\n') 19 continue 20 21 flg = True 22 for char in keyWord.strip(): 23 if char <= u'\u4e00' or char >= u'\u9fa5': 24 flg = False 25 break 26 if not flg: 27 print('Please input the Chinese Key Word for search.(Such as \'logistics\').' + '\n') 28 continue 29 30 break

PostgreSQL database processing

1 import psycopg2 4 import traceback 5 6 def connDB(_cfg): 7 try: 8 conn = psycopg2.connect(database=_cfg['servicename'], 9 user=_cfg['dbuser'], 10 password=_cfg['dbpw'], 11 host=_cfg['host'], 12 port=_cfg['port']) 13 return conn 14 except Exception: 15 printLog('E', 'Exception occur at DB Connection.' + '\n' + traceback.format_exc()) 16 17 def executeSql(_cfg, _sql): 18 try: 19 conn = connDB(_cfg) 20 cur = conn.cursor() 21 cur.execute(_sql) 22 23 results = cur.fetchall() 24 return list(map(lambda x: x[0], results)) 25 except Exception: 26 printLog('E', 'Exception occur at Execute SQL.' + '\n' + traceback.format_exc()) 27 finally: 28 cur.close() 29 conn.rollback() 30 conn.close()

Log processing

1 logging.basicConfig(filename='log_' + datetime.now().strftime('%Y%m%d') + '.txt', 2 level=logging.INFO, 3 format=' %(asctime)s - %(levelname)s - %(message)s') 4 5 logLevel = {'D': logging.DEBUG, 6 'I': logging.INFO, 7 'W': logging.WARNING, 8 'E': logging.ERROR, 9 'C': logging.CRITICAL} 10 11 def printLog(_lvl, _msg): 12 logging.log(logLevel[_lvl], _msg) 13 if logging.ERROR == logLevel[_lvl]: 14 print(_msg) 15 exit() 16 17 18 printLog('E', 'srcpath is not a exists path.') 19 printLog('I', 'Get Src Path : %s' % srcPath)

MAP function application

String truncation by a character in batch processing of list elements

1 def getPreOfNm(x): 2 if 1 < x.count('_'): 3 return x[0:x.find('_', x.find('_') + 1)] 4 else: 5 return x 6 7 # Get prefix of CRUD object name 8 prefixObjNm = list(set(map(getPreOfNm, lstTb))) 9 prefixObjNm.sort()

Catalog processing

Directory / file judgment; path partition of directory; file name acquisition of complete path;

1 # Check the srcPath 2 fullFilePaths = [] 3 if os.path.isdir(srcPath): 4 for folderName, subFolders, fileNames in os.walk(srcPath): 5 if os.path.split(folderName)[1] in ['tcs', 'doc']: continue 6 for fn in fileNames: 7 # Get src file 8 mObj = fileNmReg.search(fn) 9 if mObj: 10 fullFilePaths.append(os.path.join(folderName, fn)) 11 elif os.path.isfile(srcPath): 12 # Get src file 13 fn = os.path.basename(os.path.realpath(srcPath)) 14 mObj = fileNmReg.search(fn) 15 if mObj: 16 fullFilePaths.append(srcPath)

PDF file reading

Source: https://www.cnblogs.com/alexzhang92/p/11488949.html

1 from pdfminer.converter import TextConverter 2 from pdfminer.layout import LAParams 3 from pdfminer.pdfinterp import PDFResourceManager, process_pdf 4 import os 5 6 7 def read_pdf(pdf): 8 # resource manager 9 rsrcmgr = PDFResourceManager() 10 retstr = StringIO() 11 laparams = LAParams() 12 # device 13 device = TextConverter(rsrcmgr, retstr, laparams=laparams) 14 process_pdf(rsrcmgr, device, pdf) 15 device.close() 16 content = retstr.getvalue() 17 retstr.close() 18 # Get all rows 19 contents = str(content).split("\n") 20 21 return contents

CSV file export

1 # Init result file 2 rstFile = open(os.path.join(srcPath, '[CRUD]' + datetime.now().strftime('%Y%m%d%H%M%S') + '.csv'), 'w', newline='') 3 rstWtr = csv.writer(rstFile, delimiter='\t', lineterminator='\n') 4 # Write head 5 rstWtr.writerow(['TYPE', 'CI', 'ENCODE', 'LINE NUM', 'CRUD', 'TABLE NM', 'FULL PATH'])

Please indicate the original link for reprint, thank you.

8 May 2020, 10:04 | Views: 2425