XLSXWriter Pandas CSV Python Notes

Sun, 10 Jan 2021 00:00:00 +0000

Some useful notes in dealing with xlsxwriter, csv, and pandas thing. Still planning to add sql support for fetching data from remote source, then processing it via pandas, then save as xlsx data.

import csv
import xlsxwriter
import datetime as dt
import pandas as pd

# Create an new Excel file and add a worksheet.
workbook = xlsxwriter.Workbook('sample-xlsx.xlsx')
worksheet = workbook.add_worksheet('GPUH')
date_format = workbook.add_format({'num_format': 'mm/dd/yyyy',
 'align': 'left'})

def to_dt(date):
 return dt.datetime.strptime(date, '%m/%d/%Y')

pd_csv = pd.read_csv('sample-data.csv')

print(pd_csv)

pd_csv['Date'] = pd_csv['Date'].apply(to_dt)

print(pd_csv[['Price', 'Piece']].groupby(pd_csv['Date'].dt.isocalendar().week).sum())

with open('sample-data.csv') as csv_file:
 csv_reader = csv.reader(csv_file, delimiter=',')

 # pd_df = pd.DataFrame(ind)
 print(pd_csv)

 header = ('Date', 'Price', 'Pieces')

 # Header

 for idx, row in enumerate(csv_reader):
 if idx == 0:
 for ridx, each in enumerate(row):
 worksheet.write(idx, ridx, each)
 else:
 for ridx, col in enumerate(row):
 if ridx == 0:
 data = dt.datetime.strptime(col, '%m/%d/%Y')
 worksheet.write(idx, ridx, data, date_format)
 else:
 worksheet.write(idx, ridx, int(col))

workbook.close()

Prettytable for commandline.

from prettytable import PrettyTable
x = PrettyTable()

test_data = {
 'a1': {
 'url': 'https://sec.domain.com',
 'preset1': True,
 'preset2': True,
 'preset3': False,
 'preset4': True,
 'preset5': True,
 'preset6': False,
 'preset7': True,
 },
 'a2': {
 'url': 'https://sec.domain.com',
 'preset1': True,
 'preset2': True,
 'preset3': False,
 'preset4': True,
 'preset5': True,
 'preset6': False,
 'preset7': True,
 },
 'a3': {
 'url': 'https://sec.domain.com',
 'preset1': True,
 'preset2': True,
 'preset3': False,
 'preset4': True,
 'preset5': True,
 'preset6': False,
 'preset7': True,
 },
 'a4': {
 'url': 'https://sec.domain.com',
 'preset1': True,
 'preset2': True,
 'preset3': False,
 'preset4': True,
 'preset5': True,
 'preset6': False,
 'preset7': True,
 },
}

x.field_names = test_data['a1'].keys()
for col, cval in test_data.items():
 x.add_row(cval.values())
print(x)

Mariadb/sqlalchemy to Pandas

from sqlalchemy import create_engine, text
import pandas as pd

engine = create_engine(
 "mariadb+pymsql://user:pass@some_mariadb/dbname?charset=utf8mb4", echo=True, future=True)

with engine.connect() as conn:
 sql_query = text("select * from table")
 df = pd.read_sql(sql_query, conn, index_col="id")

############################

import pandas as pd
import mariadb

mysql_connection = mariadb.connect(host='localhost',
 user='root',
 password='',
 database='tutorfall2016',)

sql = "SELECT * FROM sign_ins"
pd.read_sql(sql, mysql_connection, index_col='id')

Selenium Python Notes

Fri, 08 Jan 2021 00:00:00 +0000

Joting note that might be useful.

Selenium Interaction with Svelte Link and Button

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait

def wait_until_element_found(xpath):
 WebDriverWait(browser, timeout=10).until(
 lambda x: browser.find_element_by_xpath(xpath)
 )

 # To handle scrolling just incase
 last_coordinate = 300
 while not browser.find_element_by_xpath(xpath).is_displayed():
 print(last_coordinate)
 browser.execute_script(
 "window.scrollTo(0, {0})".format(last_coordinate))
 last_coordinate += 300
 sleep(1)
 sleep(1)

 # For debugging and tracking
 element = browser.find_element_by_xpath(xpath)
 print(element.location)

 browser.find_element_by_xpath(xpath).click()

browser = webdriver.Firefox()
browser.maximize_window()
browser.get('https://tech.moe.ph/')
wait_until_element_found("//button[contains(@class,'cursor-pointer')]")
wait_until_element_found("//a[.='Tags']")
wait_until_element_found("//li[.='live-system']")
wait_until_element_found(
 "//h1[starts-with(., 'Using Persistent Live System as Poor Man')]")

# xpath with contains
# wait_until_element_found(
# "//h1[contains(text(), 'Persistent Live System as Poor']")
# wait_until_element_found(
# "//*[contains(text(), 'Persistent Live System as Poor']")
# Reference
# https://stackoverflow.com/questions/12323403/how-do-i-find-an-element-that-contains-specific-text-in-selenium-webdriver-pyth
# driver.find_elements_by_xpath("//*[contains(text(), 'My Button')]")

wait_until_element_found("//a[.='debian']")
wait_until_element_found(
 "//h1[starts-with(., 'Using Persistent Live System as Poor Man')]")
wait_until_element_found(
 "//h1[starts-with(., 'Using Persistent Live System as Poor Man')]")
wait_until_element_found(
 "//button[.='Back']")
browser.quit()
print("Done!")

Selenium with Form Interaction

from selenium import webdriver
# from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait, Select
from time import sleep

browser = webdriver.Firefox()
#browser = webdriver.Chrome()
browser.maximize_window()
# browser.get(
# 'file:///home/tonydeguia/Downloads/templated-industrious/elements.html')
browser.get(
 'https://templated.co/items/demos/industrious/elements.html')

def wait_until_element_found(xpath):
 # browser.execute_script("window.scrollTo(0, 0)")
 WebDriverWait(browser, timeout=10).until(
 lambda x: browser.find_element_by_xpath(xpath)
 )

 # To handle scrolling just incase
 # last_coordinate = 300
 # while not browser.find_element_by_xpath(xpath).is_displayed():
 # print(last_coordinate)
 # browser.execute_script(
 # "window.scrollTo(0, {0})".format(last_coordinate))
 # last_coordinate += 300
 # sleep(1)
 sleep(1)

 # For debugging and tracking
 element = browser.find_element_by_xpath(xpath)
 print(element.location)
 # browser.execute_script("window.scrollTo({0}, {1})"
 # .format(element.location['x'] + 20, element.location['y'] + 20))

 browser.find_element_by_xpath(xpath).click()

def wait_until_then_select(xpath, choice):
 # browser.execute_script("window.scrollTo(0, 0)")
 WebDriverWait(browser, timeout=10).until(
 lambda x: browser.find_element_by_xpath(xpath)
 )
 sleep(1)

 Select(browser.find_element_by_xpath(xpath)).select_by_value(choice)

def wait_until_then_fill(xpath, data):
 WebDriverWait(browser, timeout=10).until(
 lambda x: browser.find_element_by_xpath(xpath)
 )
 sleep(1)

 element = browser.find_element_by_xpath(xpath)
 print(element.location)

 browser.find_element_by_xpath(xpath).send_keys(data)

def wait_until_then_submit(xpath):
 WebDriverWait(browser, timeout=10).until(
 lambda x: browser.find_element_by_xpath(xpath)
 )
 sleep(1)

 element = browser.find_element_by_xpath(xpath)
 print(element.location)

 browser.find_element_by_xpath(xpath).submit()

# select by value
wait_until_then_select("//select[@id='category']", "epsilon")

# Checkbox button
# Use element instead of id for checkbox
wait_until_element_found("//label[.='Checkbox alpha']")
wait_until_element_found("//label[.='Checkbox beta']")

# Check if True
# driver.element().is_selected()

# Radio select
# Use element instead of id for radio select
wait_until_element_found("//label[.='Radio gamma']")

# Text field select
wait_until_then_fill("//input[@name='name']", "superusername")
wait_until_then_fill("//input[@name='email']", "superemailaddress")

# Submit
wait_until_then_submit("//input[@value='Submit Form']")

sleep(10)
browser.quit()
print("Done!")

browser.get('https://www.w3schools.com/bootstrap/bootstrap_modal.asp')
wait_until_element_found("//button[.='Click To Open Modal']")
browser.switch_to.active_element
wait_until_element_found("//p[.='Some text in the modal.']")
wait_until_element_found("//button[.='Close']")

Extended Selenium Class

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait, Select
from time import sleep
import sys

class WebNavigator(webdriver.Firefox, webdriver.Chrome):
 sleep_int = 0.5
 webdriver_timeout = 10

 def __init__(self, browser):
 if browser == "Firefox":
 webdriver.Firefox.__init__(self)
 elif browser == "Chrome":
 webdriver.Chrome.__init__(self)
 else:
 sys.exit("Please specify webdriver (Firefox or Chrome).")
 self.maximize_window()

 def wait_then_click(self, xpath):
 WebDriverWait(self, timeout=self.webdriver_timeout).until(
 lambda x: self.find_element_by_xpath(xpath)
 )
 self.find_element_by_xpath(xpath).click()
 sleep(self.sleep_int)

 def wait_then_select(self, xpath, choice):
 WebDriverWait(self, timeout=self.webdriver_timeout).until(
 lambda x: self.find_element_by_xpath(xpath)
 )
 Select(self.find_element_by_xpath(xpath)).select_by_value(choice)
 sleep(self.sleep_int)

 def wait_then_fill(self, xpath, data):
 WebDriverWait(self, timeout=self.webdriver_timeout).until(
 lambda x: self.find_element_by_xpath(xpath)
 )
 self.find_element_by_xpath(xpath).send_keys(data)
 sleep(self.sleep_int)

Assorted helper for Selenium

import argparse

parser = argparse.ArgumentParser(description="""
 Parser
 """)
parser.add_argument("username", help="Account login")
parser.add_argument("password", help="Account password")
parser.add_argument("-b", "--browser",
 help="Webdriver to be used. Default: Chrome")
group = parser.add_mutually_exclusive_group()
group.add_argument(
 "-a", "--all", help="Execute to all regions", action="store_true")
group.add_argument("-r", "--region", help="Execute to this specific region")
args = parser.parse_args()
print(args.username)
print(args.password)
print(args.browser)

Selenium CSS Selector

[HTML TAG].[CLASS] [HTML TAG] [HTML TAG][ATTRIBUTE]

wait_until_css("div.row form label[for='checkbox-alpha']")
wait_until_css("div.row form label[for='checkbox-beta']")
wait_until_css_select("div.row form #category", "epsilon")

Python | blog.moe.ph