commit fd4b2fd37a82c4015a171933fd6337201bd67798 Author: Tobias Eidelpes Date: Sat Feb 24 15:16:00 2024 +0100 Initial commit diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..4a4726a --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use_nix diff --git a/Zacks.side b/Zacks.side new file mode 100644 index 0000000..233b837 --- /dev/null +++ b/Zacks.side @@ -0,0 +1,348 @@ +{ + "id": "ffeb5101-a99c-4ce8-919f-02116dfc255e", + "version": "2.0", + "name": "Zacks", + "url": "https://www.zacks.com", + "tests": [{ + "id": "b7a9f639-f694-4c23-ac23-3ea3449603d6", + "name": "Download Screen", + "commands": [{ + "id": "13ebee17-c1dc-4e38-87ce-2fa123ca6ee2", + "comment": "", + "command": "open", + "target": "/screening/stock-screener", + "targets": [], + "value": "" + }, { + "id": "761f50ff-5404-42f1-a93d-819a3ceff00b", + "comment": "", + "command": "setWindowSize", + "target": "1118x1366", + "targets": [], + "value": "" + }, { + "id": "57c20488-4291-45cf-bd5a-19fd83f3cb68", + "comment": "", + "command": "selectFrame", + "target": "index=0", + "targets": [ + ["index=0"] + ], + "value": "" + }, { + "id": "0afb4792-ede2-4d40-a27c-60ccbd7414b4", + "comment": "", + "command": "click", + "target": "id=val_12010", + "targets": [ + ["id=val_12010", "id"], + ["css=#val_12010", "css:finder"], + ["xpath=//input[@id='val_12010']", "xpath:attributes"], + ["xpath=//div[@id='regular']/table/tbody/tr[9]/th/fieldset/span[2]/input", "xpath:idRelative"], + ["xpath=//tr[9]/th/fieldset/span[2]/input", "xpath:position"] + ], + "value": "" + }, { + "id": "04ff45d4-95a2-44f0-8dfe-6cf7c52af3ef", + "comment": "", + "command": "type", + "target": "id=val_12010", + "targets": [ + ["id=val_12010", "id"], + ["css=#val_12010", "css:finder"], + ["xpath=//input[@id='val_12010']", "xpath:attributes"], + ["xpath=//div[@id='regular']/table/tbody/tr[9]/th/fieldset/span[2]/input", "xpath:idRelative"], + ["xpath=//tr[9]/th/fieldset/span[2]/input", "xpath:position"] + ], + "value": "3000" + }, { + "id": "ac789578-8fad-4378-9033-40cd581db9ac", + "comment": "", + "command": "click", + "target": "css=.add_criteria_12010", + "targets": [ + ["css=.add_criteria_12010", "css:finder"], + ["xpath=(//a[contains(text(),'Add')])[2]", "xpath:link"], + ["xpath=//a[@onclick=\"getCriteriaParam(10000,8,12010,'U');\"]", "xpath:attributes"], + ["xpath=//div[@id='regular']/table/tbody/tr[9]/td/a", "xpath:idRelative"], + ["xpath=(//a[contains(@href, 'javascript:void(0);')])[20]", "xpath:href"], + ["xpath=//tr[9]/td/a", "xpath:position"] + ], + "value": "" + }, { + "id": "8f852015-3e8b-4b4b-8aa6-7b41ef4a54c3", + "comment": "", + "command": "click", + "target": "id=val_11015", + "targets": [ + ["id=val_11015", "id"], + ["css=#val_11015", "css:finder"], + ["xpath=//select[@id='val_11015']", "xpath:attributes"], + ["xpath=//div[@id='regular']/table/tbody/tr[13]/th/fieldset/span[2]/select", "xpath:idRelative"], + ["xpath=//tr[13]/th/fieldset/span[2]/select", "xpath:position"] + ], + "value": "" + }, { + "id": "48958c17-490a-4c79-99d6-c6fb3f3e7b26", + "comment": "", + "command": "select", + "target": "id=val_11015", + "targets": [], + "value": "label=YES" + }, { + "id": "dc57fc87-0e59-4474-b43d-fd27026cf099", + "comment": "", + "command": "click", + "target": "css=#val_11015 > option:nth-child(3)", + "targets": [ + ["css=#val_11015 > option:nth-child(3)", "css:finder"], + ["xpath=//option[@value='YES']", "xpath:attributes"], + ["xpath=//select[@id='val_11015']/option[3]", "xpath:idRelative"], + ["xpath=//tr[13]/th/fieldset/span[2]/select/option[3]", "xpath:position"], + ["xpath=//option[contains(.,'YES')]", "xpath:innerText"] + ], + "value": "" + }, { + "id": "0a5f2d41-f331-49e1-a106-5026a44dd803", + "comment": "", + "command": "click", + "target": "css=.add_criteria_11015", + "targets": [ + ["css=.add_criteria_11015", "css:finder"], + ["xpath=(//a[contains(text(),'Add')])[6]", "xpath:link"], + ["xpath=//a[@onclick=\"getCriteriaParam(10000,12,11015,'H');\"]", "xpath:attributes"], + ["xpath=//div[@id='regular']/table/tbody/tr[13]/td/a", "xpath:idRelative"], + ["xpath=(//a[contains(@href, 'javascript:void(0);')])[25]", "xpath:href"], + ["xpath=//tr[13]/td/a", "xpath:position"] + ], + "value": "" + }, { + "id": "f3440f83-4934-4c02-b838-73a8ac814c5b", + "comment": "", + "command": "click", + "target": "id=edit-view-tab", + "targets": [ + ["id=edit-view-tab", "id"], + ["css=#edit-view-tab", "css:finder"], + ["xpath=//button[@id='edit-view-tab']", "xpath:attributes"], + ["xpath=//button[4]", "xpath:position"] + ], + "value": "" + }, { + "id": "213ad728-e019-4113-9f23-408e124f0333", + "comment": "", + "command": "click", + "target": "linkText=Company Descriptors", + "targets": [ + ["linkText=Company Descriptors", "linkText"], + ["css=#criteria_cat_11000 > a", "css:finder"], + ["xpath=//a[contains(text(),'Company Descriptors')]", "xpath:link"], + ["xpath=//li[@id='criteria_cat_11000']/a", "xpath:idRelative"], + ["xpath=(//a[contains(@href, 'javascript:void(0);')])[2]", "xpath:href"], + ["xpath=//li[2]/a", "xpath:position"], + ["xpath=//a[contains(.,'Company Descriptors')]", "xpath:innerText"] + ], + "value": "" + }, { + "id": "a4173ee5-9972-4083-98ff-db684e5d57de", + "comment": "", + "command": "click", + "target": "id=editView11020", + "targets": [ + ["id=editView11020", "id"], + ["css=#editView11020", "css:finder"], + ["xpath=//input[@id='editView11020']", "xpath:attributes"], + ["xpath=(//form[@id='editform']/span/input)[4]", "xpath:idRelative"], + ["xpath=//tr[4]/th/fieldset/span/form/span/input", "xpath:position"] + ], + "value": "" + }, { + "id": "45c8ac84-40d5-4c14-8965-8fd3aba1c221", + "comment": "", + "command": "click", + "target": "id=editView11025", + "targets": [ + ["id=editView11025", "id"], + ["css=#editView11025", "css:finder"], + ["xpath=//input[@id='editView11025']", "xpath:attributes"], + ["xpath=(//form[@id='editform']/span/input)[5]", "xpath:idRelative"], + ["xpath=//tr[5]/th/fieldset/span/form/span/input", "xpath:position"] + ], + "value": "" + }, { + "id": "4116703c-e0ff-4119-9c0b-473f005ba459", + "comment": "", + "command": "click", + "target": "id=editView11030", + "targets": [ + ["id=editView11030", "id"], + ["css=#editView11030", "css:finder"], + ["xpath=//input[@id='editView11030']", "xpath:attributes"], + ["xpath=(//form[@id='editform']/span/input)[6]", "xpath:idRelative"], + ["xpath=//tr[6]/th/fieldset/span/form/span/input", "xpath:position"] + ], + "value": "" + }, { + "id": "b1105820-1baa-4407-8e52-f7ce4d5ed4b6", + "comment": "", + "command": "click", + "target": "id=editView11005", + "targets": [ + ["id=editView11005", "id"], + ["name=check_value[]", "name"], + ["css=#editView11005", "css:finder"], + ["xpath=//input[@id='editView11005']", "xpath:attributes"], + ["xpath=//form[@id='editform']/span/input", "xpath:idRelative"], + ["xpath=//form/span/input", "xpath:position"] + ], + "value": "" + }, { + "id": "983c4a02-9074-4f8e-8189-1ef8e385e09a", + "comment": "", + "command": "click", + "target": "id=criteria_cat_17000", + "targets": [ + ["id=criteria_cat_17000", "id"], + ["css=#criteria_cat_17000", "css:finder"], + ["xpath=//li[@id='criteria_cat_17000']", "xpath:attributes"], + ["xpath=//ul[@id='tabLinkUL']/li[7]", "xpath:idRelative"], + ["xpath=//li[7]", "xpath:position"], + ["xpath=//li[contains(.,'EPS Surprises & Actuals')]", "xpath:innerText"] + ], + "value": "" + }, { + "id": "f5689219-0bde-4cdc-85df-908bf22256ca", + "comment": "", + "command": "click", + "target": "id=criteria_cat_14000", + "targets": [ + ["id=criteria_cat_14000", "id"], + ["css=#criteria_cat_14000", "css:finder"], + ["xpath=//li[@id='criteria_cat_14000']", "xpath:attributes"], + ["xpath=//ul[@id='tabLinkUL']/li[4]", "xpath:idRelative"], + ["xpath=//li[4]", "xpath:position"], + ["xpath=//li[contains(.,'Price & Price Changes')]", "xpath:innerText"] + ], + "value": "" + }, { + "id": "f86f4c12-c0b5-485c-b432-a50ace9be4ec", + "comment": "", + "command": "click", + "target": "id=editView14005", + "targets": [ + ["id=editView14005", "id"], + ["name=check_value[]", "name"], + ["css=#editView14005", "css:finder"], + ["xpath=//input[@id='editView14005']", "xpath:attributes"], + ["xpath=//form[@id='editform']/span/input", "xpath:idRelative"], + ["xpath=//form/span/input", "xpath:position"] + ], + "value": "" + }, { + "id": "67a053b6-d3cf-45f4-b5b9-75e5aced6f25", + "comment": "", + "command": "click", + "target": "linkText=EPS Surprises & Actuals", + "targets": [ + ["linkText=EPS Surprises & Actuals", "linkText"], + ["css=#criteria_cat_17000 > a", "css:finder"], + ["xpath=//a[contains(text(),'EPS Surprises & Actuals')]", "xpath:link"], + ["xpath=//li[@id='criteria_cat_17000']/a", "xpath:idRelative"], + ["xpath=(//a[contains(@href, 'javascript:void(0);')])[7]", "xpath:href"], + ["xpath=//li[7]/a", "xpath:position"], + ["xpath=//a[contains(.,'EPS Surprises & Actuals')]", "xpath:innerText"] + ], + "value": "" + }, { + "id": "59f33a0f-c599-49ee-8d2e-bfa3a9ee7060", + "comment": "", + "command": "click", + "target": "linkText=EPS Estimates", + "targets": [ + ["linkText=EPS Estimates", "linkText"], + ["css=#criteria_cat_19000 > a", "css:finder"], + ["xpath=//a[contains(text(),'EPS Estimates')]", "xpath:link"], + ["xpath=//li[@id='criteria_cat_19000']/a", "xpath:idRelative"], + ["xpath=(//a[contains(@href, 'javascript:void(0);')])[9]", "xpath:href"], + ["xpath=//li[9]/a", "xpath:position"], + ["xpath=//a[contains(.,'EPS Estimates')]", "xpath:innerText"] + ], + "value": "" + }, { + "id": "8a2389f9-6bda-4f85-ae80-98cdf2fa9a33", + "comment": "", + "command": "click", + "target": "id=editView19045", + "targets": [ + ["id=editView19045", "id"], + ["css=#editView19045", "css:finder"], + ["xpath=//input[@id='editView19045']", "xpath:attributes"], + ["xpath=(//form[@id='editform']/span/input)[9]", "xpath:idRelative"], + ["xpath=//tr[9]/th/fieldset/span/form/span/input", "xpath:position"] + ], + "value": "" + }, { + "id": "fec893ec-e3ef-4c17-96b6-3035d90a658d", + "comment": "", + "command": "click", + "target": "id=editView19055", + "targets": [ + ["id=editView19055", "id"], + ["css=#editView19055", "css:finder"], + ["xpath=//input[@id='editView19055']", "xpath:attributes"], + ["xpath=(//form[@id='editform']/span/input)[11]", "xpath:idRelative"], + ["xpath=//tr[11]/th/fieldset/span/form/span/input", "xpath:position"] + ], + "value": "" + }, { + "id": "3f27420b-6749-4411-8bec-38181ff0b3fb", + "comment": "", + "command": "click", + "target": "id=editView19070", + "targets": [ + ["id=editView19070", "id"], + ["css=#editView19070", "css:finder"], + ["xpath=//input[@id='editView19070']", "xpath:attributes"], + ["xpath=(//form[@id='editform']/span/input)[14]", "xpath:idRelative"], + ["xpath=//tr[14]/th/fieldset/span/form/span/input", "xpath:position"] + ], + "value": "" + }, { + "id": "3264acc5-2016-4809-977b-0a95d4eaad54", + "comment": "", + "command": "click", + "target": "id=run_screen_result", + "targets": [ + ["id=run_screen_result", "id"], + ["css=#run_screen_result", "css:finder"], + ["xpath=//button[@id='run_screen_result']", "xpath:attributes"], + ["xpath=//form[@id='criteria_form']/div/div[3]/button", "xpath:idRelative"], + ["xpath=//div[3]/button", "xpath:position"], + ["xpath=//button[contains(.,'Run Screen')]", "xpath:innerText"] + ], + "value": "" + }, { + "id": "5a4af6c9-c23f-4fc4-b883-8a75d821d28b", + "comment": "", + "command": "click", + "target": "css=.buttons-csv > span", + "targets": [ + ["css=.buttons-csv > span", "css:finder"], + ["xpath=//div[@id='screener_table_wrapper']/div/a/span", "xpath:idRelative"], + ["xpath=//div[2]/div/div/div/a/span", "xpath:position"], + ["xpath=//span[contains(.,'CSV')]", "xpath:innerText"] + ], + "value": "" + }] + }], + "suites": [{ + "id": "2ce02e75-e985-4374-a129-4dcb2ff4e52e", + "name": "Default Suite", + "persistSession": false, + "parallel": false, + "timeout": 300, + "tests": ["b7a9f639-f694-4c23-ac23-3ea3449603d6"] + }], + "urls": ["https://www.zacks.com/"], + "plugins": [] +} \ No newline at end of file diff --git a/commitments.py b/commitments.py new file mode 100644 index 0000000..2e940bf --- /dev/null +++ b/commitments.py @@ -0,0 +1,129 @@ +import pandas as pd +import numpy as np +import argparse + + +def select_cols(df): + columns = [ + "Open_Interest_All", + "M_Money_Positions_Long_ALL", + "M_Money_Positions_Short_ALL", + ] + return pd.DataFrame(df, columns=columns).reset_index(drop=True) + + +def read_excel(excel_path): + df = pd.read_excel(excel_path) + relevant_dfs = [] + crude_oil_wti = relevant_dfs.append( + df.loc[ + df["Market_and_Exchange_Names"] + == "CRUDE OIL, LIGHT SWEET-WTI - ICE FUTURES EUROPE" + ] + ) + crude_oil_brent = relevant_dfs.append( + df.loc[ + ( + df["Market_and_Exchange_Names"] + == "BRENT CRUDE OIL LAST DAY - NEW YORK MERCANTILE EXCHANGE" + ) + | ( + df["Market_and_Exchange_Names"] + == "BRENT LAST DAY - NEW YORK MERCANTILE EXCHANGE" + ) + ] + ) + heating_oil = relevant_dfs.append( + df.loc[ + ( + df["Market_and_Exchange_Names"] + == "#2 HEATING OIL- NY HARBOR-ULSD - NEW YORK MERCANTILE EXCHANGE" + ) + | ( + df["Market_and_Exchange_Names"] + == "NY HARBOR ULSD - NEW YORK MERCANTILE EXCHANGE" + ) + ] + ) + palladium = relevant_dfs.append( + df.loc[ + df["Market_and_Exchange_Names"] + == "PALLADIUM - NEW YORK MERCANTILE EXCHANGE" + ] + ) + platinum = relevant_dfs.append( + df.loc[ + df["Market_and_Exchange_Names"] == "PLATINUM - NEW YORK MERCANTILE EXCHANGE" + ] + ) + silver = relevant_dfs.append( + df.loc[df["Market_and_Exchange_Names"] == "SILVER - COMMODITY EXCHANGE INC."] + ) + gold = relevant_dfs.append( + df.loc[df["Market_and_Exchange_Names"] == "GOLD - COMMODITY EXCHANGE INC."] + ) + copper = relevant_dfs.append( + df.loc[ + ( + df["Market_and_Exchange_Names"] + == "COPPER-GRADE #1 - COMMODITY EXCHANGE INC." + ) + | ( + df["Market_and_Exchange_Names"] + == "COPPER- #1 - COMMODITY EXCHANGE INC." + ) + ] + ) + aluminium = relevant_dfs.append( + df.loc[ + ( + df["Market_and_Exchange_Names"] + == "ALUMINUM MW US TR PLATTS - COMMODITY EXCHANGE INC." + ) + | ( + df["Market_and_Exchange_Names"] + == "ALUMINUM MWP - COMMODITY EXCHANGE INC." + ) + ] + ) + steel = relevant_dfs.append( + df.loc[ + ( + df["Market_and_Exchange_Names"] + == "US MIDWEST DOMESTIC HOT-ROLL - COMMODITY EXCHANGE INC." + ) + | (df["Market_and_Exchange_Names"] == "STEEL-HRC - COMMODITY EXCHANGE INC.") + ] + ) + natural_gas = relevant_dfs.append( + df.loc[ + ( + df["Market_and_Exchange_Names"] + == "NATURAL GAS - NEW YORK MERCANTILE EXCHANGE" + ) + | ( + df["Market_and_Exchange_Names"] + == "NAT GAS NYME - NEW YORK MERCANTILE EXCHANGE" + ) + ] + ) + return relevant_dfs + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + "commitments", description="Create CSV file for commitments of traders report" + ) + parser.add_argument( + "filename", + help="Path to Excel file for a particular year of commitments of traders", + ) + parser.add_argument( + "--dest", default="c_year.csv", help="Location to save parsed CSV to" + ) + args = parser.parse_args() + dfs = read_excel(args.filename) + relevant_dfs = [select_cols(df) for df in dfs] + # Stack horizontally + df = pd.concat(relevant_dfs, axis=1).iloc[::-1] + df.to_csv(args.dest, index=False) diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..11d0f30 --- /dev/null +++ b/shell.nix @@ -0,0 +1,23 @@ +{ nixpkgs ? import {} }: + +with nixpkgs; + +stdenv.mkDerivation rec { + name = "python-virtualenv-shell"; + env = buildEnv { name = name; paths = buildInputs; }; + buildInputs = [ + python311 + python311Packages.virtualenv + python311Packages.pandas + python311Packages.xlrd + python311Packages.python-lsp-server + python311Packages.python-lsp-black + # libGL + # glib + ]; + shellHook = '' + # set SOURCE_DATE_EPOCH so that we can use python wheels + SOURCE_DATE_EPOCH=$(date +%s) + ''; + LD_LIBRARY_PATH = "$LD_LIBRARY_PATH:pkgs.stdenv.cc.cc.lib/lib:pkgs.glib.out/lib:pkgs.libGL/lib"; +} diff --git a/zacks.py b/zacks.py new file mode 100644 index 0000000..f37ccf0 --- /dev/null +++ b/zacks.py @@ -0,0 +1,75 @@ +import pandas as pd +import argparse +from datetime import datetime + + +def remove_otc(df): + return df[df["Exchange"] != "OTC"] + + +def reorder_cols(df): + return df[ + [ + "Company Name", + "Ticker", + "Exchange", + "Sector", + "Industry", + "Month of Fiscal Yr End", + "Market Cap (mil)", + "Last Close", + "F0 Consensus Est.", + "F1 Consensus Est.", + "F2 Consensus Est.", + ] + ] + + +def rename_cols(df): + return df.rename( + columns={ + "F0 Consensus Est.": "EPS0", + "F1 Consensus Est.": "EPS1", + "F2 Consensus Est.": "EPS2", + } + ) + +def add_cols(df): + df['EG1'] = ((df['EPS1'] - df['EPS0']) / abs(df['EPS1'])).round(4) + df['EG2'] = ((df['EPS2'] - df['EPS1']) / abs(df['EPS2'])).round(4) + df['PE0'] = (df['Last Close'] / df['EPS0']).round(2) + df['PE1'] = (df['Last Close'] / df['EPS1']).round(2) + df['PE2'] = (df['Last Close'] / df['EPS2']).round(2) + df['PEG1'] = (df['PE1'] / (df['EG1'] * 100)).round(2) + df['PEG2'] = (df['PE2'] / (df['EG2'] * 100)).round(2) + df['EG1 Mean'] = df.groupby('Industry')['EG1'].transform('mean').round(4) + df['EG2 Mean'] = df.groupby('Industry')['EG2'].transform('mean').round(4) + df['PE0 Mean'] = df.groupby('Industry')['PE0'].transform('mean').round(2) + df['PE1 Mean'] = df.groupby('Industry')['PE1'].transform('mean').round(2) + df['PE2 Mean'] = df.groupby('Industry')['PE2'].transform('mean').round(2) + return df + + +if __name__ == "__main__": + today = datetime.today() + iso_date = today.strftime("%Y-%m-%d") + parser = argparse.ArgumentParser( + "zacks-screen", description="Create cleaned CSV file from zacks screen" + ) + parser.add_argument( + "filename", + help="Path to zacks screen CSV file", + ) + parser.add_argument( + "--dest", + default=f"zacks-screen-{iso_date}.csv", + help="Location to save parsed CSV to", + ) + args = parser.parse_args() + df = pd.read_csv(args.filename) + df = remove_otc(df) + df = reorder_cols(df) + df = rename_cols(df) + df = add_cols(df) + df.to_csv(args.dest, index=False) + print(df)