Commit @10bd22275e455a47ebe2e5829ba4abc52a3ca6ea

윤영준 2023-10-22

also, weather data update now supports DB

@10bd22275e455a47ebe2e5829ba4abc52a3ca6ea

2d20329

10bd222

tools/weather_agency_api/weather_api.py

--- tools/weather_agency_api/weather_api.py

+++ tools/weather_agency_api/weather_api.py


 import os
+import numpy as np
 import pandas as pd
 import requests
+import psycopg2
 from datetime import datetime, timedelta
 from io import StringIO
 from tools.weather_agency_api.duplicate_check import duplicate_check
 from tools.weather_agency_api.check_missing import check_missing
 from sqlalchemy import create_engine
 
-DATABASE_URL = "postgresql+psycopg2://username:ts4430!@@localhost:5432/welding"
+def buck_equation(temperature): # temp in Celsius
+    saturation_vapor_pressure = 0.61121 * np.exp((18.678-temperature/234.5)*(temperature/(257.14+temperature)))
+    return saturation_vapor_pressure * 1000 # KPa -> Pa
+
+def absolute_humidity(relative_humidity, temperature):
+    relative_humidity = np.array(relative_humidity)
+    temperature = np.array(temperature)
+    saturation_vapor_pressure = buck_equation(temperature)
+    # 461.5/Kg Kelvin is specific gas constatnt
+    return saturation_vapor_pressure * relative_humidity * 0.01 /(461.5 * (temperature + 273.15) ) # g/m^3
+
+db_info ={
+    'dbname': 'welding',
+    'user': 'postgres',
+    'password': 'ts4430!@',
+    'host': 'localhost',  # e.g., 'localhost'
+    'port': '5432',  # e.g., '5432'
+}
+
 # https://apihub.kma.go.kr/ 참고
 weather_api_columns = [
     "관측시각",

     Updates the weather information up to today at 00:00.
     """
 
-    # Set up a connection using SQLAlchemy (for Pandas operations)
-    engine = create_engine(DATABASE_URL)
+    # Set up a connection using psycopg2
+    conn = psycopg2.connect(**db_info)
+
+    # Create a cursor object
+    cursor = conn.cursor()
 
     # Load existing data
     query = "SELECT * FROM weather_data"
-    df_weather_existing = pd.read_sql(query, engine)
+    df_weather_existing = pd.read_sql(query, conn)
 
     # Find the last date available in the database
-    last_date_str = f'{df_weather_existing.iloc[-1]["관측시각"]}'
-    last_date = datetime.strptime(last_date_str, '%Y%m%d%H%M')
+    last_date = df_weather_existing.iloc[-1].loc['time']
 
     # Get today's date at 00:00
     today_00 = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)

                 end_day = date + k
             else:
                 end_day = len(date_lists) - 1
-            text = call_administration_of_weather_api(date_lists[date], date_lists[end_day])  # Ensure this function is defined
+            text = call_administration_of_weather_api(date_lists[date], date_lists[end_day])
             buffer = StringIO(text)
-            df = pd.read_csv(buffer, skiprows=2, skipfooter=1, sep=r"\s+", header=None, index_col=False, engine="python").iloc[2:, :-1]
-            df = df.set_axis(weather_api_columns, axis=1, inplace=False)  # Ensure 'weather_api_columns' is defined
-            if not check_missing(df):  # Ensure this function is defined
+            df = pd.read_csv(buffer, skiprows=2, skipfooter=1, sep=r"\s+", header=None, index_col=False,
+                             engine="python").iloc[2:, :-1]
+            df = df.set_axis(weather_api_columns, axis=1, inplace=False)
+            if not check_missing(df):
                 print("API is not working!")
                 return {
                     "responses": 500
                 }
             df_weather_new = pd.concat([df_weather_new, df], ignore_index=True)
+            df_weather_new = df_weather_new.loc[:,["관측시각", "기온", "상대습도"]]
+            df_weather_new['관측시각'] = pd.to_datetime(df_weather_new['관측시각'], format='%Y%m%d%H%M')
+            df_weather_new = df_weather_new.rename(columns={
+                '관측시각': 'time',
+                '기온': 'temperature',
+                '상대습도': 'relative_humidity',
+            },
+                inplace=False
+            )
+            df_weather_new['absolute_humidity'] = absolute_humidity(df_weather_new['relative_humidity'].astype(float), df_weather_new['temperature'].astype(float))
+
             print(f"{i}/{len(range(0, len(date_lists) - 1, k)) - 1}")
 
         # Append the new data to the existing data in the database
-        df_weather_new.to_sql('weather_data', engine, if_exists='append', index=False)
+        for _, row in df_weather_new.iterrows():
+            columns = ','.join(row.keys())
+            values = ','.join([f"'{item}'" for item in row])
+            cursor.execute(f"INSERT INTO weather_data ({columns}) VALUES ({values})")
+
+        conn.commit()
     else:
         print("Weather data is already up-to-date!")
+
+    # Close the cursor and the connection
+    cursor.close()
+    conn.close()
+
 
 def update_weather_info_to_today_csv(file_name):
     """

             print(f"{i}/{len(range(0, len(date_lists) - 1, k)) - 1}")
 
         # Append the new data to the existing data
+        df_weather_new['관측시각'] = df_weather_new['관측시각'].apply(lambda x: datetime.strptime(f"{x}", '%Y%m%d%H%M'))
+        df_weather_upload = df_weather_new['관측시간','기온','상대습도']
+
         df_weather_updated = pd.concat([df_weather_existing, df_weather_new], ignore_index=True)
         df_weather_updated = duplicate_check(df_weather_updated)
         df_weather_updated.to_csv(file_name, index=False)

 
 if __name__ == "__main__":
     file = "202007010000_202308310000.csv"
-    update_weather_info_to_today(file)
+    update_weather_info_to_today()

Add a comment

Open 0
Closed 0

List

...	...	@@ -1,13 +1,33 @@
1	1	import os
	2	+import numpy as np
2	3	import pandas as pd
3	4	import requests
	5	+import psycopg2
4	6	from datetime import datetime, timedelta
5	7	from io import StringIO
6	8	from tools.weather_agency_api.duplicate_check import duplicate_check
7	9	from tools.weather_agency_api.check_missing import check_missing
8	10	from sqlalchemy import create_engine
9	11
10		-DATABASE_URL = "postgresql+psycopg2://username:ts4430!@@localhost:5432/welding"
	12	+def buck_equation(temperature): # temp in Celsius
	13	+ saturation_vapor_pressure = 0.61121 * np.exp((18.678-temperature/234.5)*(temperature/(257.14+temperature)))
	14	+ return saturation_vapor_pressure * 1000 # KPa -> Pa
	15	+
	16	+def absolute_humidity(relative_humidity, temperature):
	17	+ relative_humidity = np.array(relative_humidity)
	18	+ temperature = np.array(temperature)
	19	+ saturation_vapor_pressure = buck_equation(temperature)
	20	+ # 461.5/Kg Kelvin is specific gas constatnt
	21	+ return saturation_vapor_pressure * relative_humidity * 0.01 /(461.5 * (temperature + 273.15) ) # g/m^3
	22	+
	23	+db_info ={
	24	+ 'dbname': 'welding',
	25	+ 'user': 'postgres',
	26	+ 'password': 'ts4430!@',
	27	+ 'host': 'localhost', # e.g., 'localhost'
	28	+ 'port': '5432', # e.g., '5432'
	29	+}
	30	+
11	31	# https://apihub.kma.go.kr/ 참고
12	32	weather_api_columns = [
13	33	"관측시각",
...	...	@@ -88,16 +108,18 @@
88	108	Updates the weather information up to today at 00:00.
89	109	"""
90	110
91		- # Set up a connection using SQLAlchemy (for Pandas operations)
92		- engine = create_engine(DATABASE_URL)
	111	+ # Set up a connection using psycopg2
	112	+ conn = psycopg2.connect(**db_info)
	113	+
	114	+ # Create a cursor object
	115	+ cursor = conn.cursor()
93	116
94	117	# Load existing data
95	118	query = "SELECT * FROM weather_data"
96		- df_weather_existing = pd.read_sql(query, engine)
	119	+ df_weather_existing = pd.read_sql(query, conn)
97	120
98	121	# Find the last date available in the database
99		- last_date_str = f'{df_weather_existing.iloc[-1]["관측시각"]}'
100		- last_date = datetime.strptime(last_date_str, '%Y%m%d%H%M')
	122	+ last_date = df_weather_existing.iloc[-1].loc['time']
101	123
102	124	# Get today's date at 00:00
103	125	today_00 = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
...	...	@@ -116,22 +138,44 @@
116	138	end_day = date + k
117	139	else:
118	140	end_day = len(date_lists) - 1
119		- text = call_administration_of_weather_api(date_lists[date], date_lists[end_day]) # Ensure this function is defined
	141	+ text = call_administration_of_weather_api(date_lists[date], date_lists[end_day])
120	142	buffer = StringIO(text)
121		- df = pd.read_csv(buffer, skiprows=2, skipfooter=1, sep=r"\s+", header=None, index_col=False, engine="python").iloc[2:, :-1]
122		- df = df.set_axis(weather_api_columns, axis=1, inplace=False) # Ensure 'weather_api_columns' is defined
123		- if not check_missing(df): # Ensure this function is defined
	143	+ df = pd.read_csv(buffer, skiprows=2, skipfooter=1, sep=r"\s+", header=None, index_col=False,
	144	+ engine="python").iloc[2:, :-1]
	145	+ df = df.set_axis(weather_api_columns, axis=1, inplace=False)
	146	+ if not check_missing(df):
124	147	print("API is not working!")
125	148	return {
126	149	"responses": 500
127	150	}
128	151	df_weather_new = pd.concat([df_weather_new, df], ignore_index=True)
	152	+ df_weather_new = df_weather_new.loc[:,["관측시각", "기온", "상대습도"]]
	153	+ df_weather_new['관측시각'] = pd.to_datetime(df_weather_new['관측시각'], format='%Y%m%d%H%M')
	154	+ df_weather_new = df_weather_new.rename(columns={
	155	+ '관측시각': 'time',
	156	+ '기온': 'temperature',
	157	+ '상대습도': 'relative_humidity',
	158	+ },
	159	+ inplace=False
	160	+ )
	161	+ df_weather_new['absolute_humidity'] = absolute_humidity(df_weather_new['relative_humidity'].astype(float), df_weather_new['temperature'].astype(float))
	162	+
129	163	print(f"{i}/{len(range(0, len(date_lists) - 1, k)) - 1}")
130	164
131	165	# Append the new data to the existing data in the database
132		- df_weather_new.to_sql('weather_data', engine, if_exists='append', index=False)
	166	+ for _, row in df_weather_new.iterrows():
	167	+ columns = ','.join(row.keys())
	168	+ values = ','.join([f"'{item}'" for item in row])
	169	+ cursor.execute(f"INSERT INTO weather_data ({columns}) VALUES ({values})")
	170	+
	171	+ conn.commit()
133	172	else:
134	173	print("Weather data is already up-to-date!")
	174	+
	175	+ # Close the cursor and the connection
	176	+ cursor.close()
	177	+ conn.close()
	178	+
135	179
136	180	def update_weather_info_to_today_csv(file_name):
137	181	"""
...	...	@@ -178,6 +222,9 @@
178	222	print(f"{i}/{len(range(0, len(date_lists) - 1, k)) - 1}")
179	223
180	224	# Append the new data to the existing data
	225	+ df_weather_new['관측시각'] = df_weather_new['관측시각'].apply(lambda x: datetime.strptime(f"{x}", '%Y%m%d%H%M'))
	226	+ df_weather_upload = df_weather_new['관측시간','기온','상대습도']
	227	+
181	228	df_weather_updated = pd.concat([df_weather_existing, df_weather_new], ignore_index=True)
182	229	df_weather_updated = duplicate_check(df_weather_updated)
183	230	df_weather_updated.to_csv(file_name, index=False)
...	...	@@ -187,4 +234,4 @@
187	234
188	235	if __name__ == "__main__":
189	236	file = "202007010000_202308310000.csv"
190		- update_weather_info_to_today(file)
	237	+ update_weather_info_to_today()

Delete comment