Skip to content

Commit b6c1a90

Browse files
YueLi-0816YueLi-0816
authored
update the file dir
1 parent 7f05aaa commit b6c1a90

5 files changed

+861
-71
lines changed

‎notebook/GSV_semantic segmentation.ipynb

+14-5
Original file line numberDiff line numberDiff line change
@@ -369,15 +369,21 @@
369369
"source": [
370370
"import pandas as pd\n",
371371
"\n",
372-
"SCOOT_df = pd.read_csv('../20211028_SCOOT_Data/JupyterNotebook_CSV/useful_detector_530.csv')\n",
372+
"# if you download the raw traffic flow data via Traffic_flow_Download.ipynb and pre-process the data via Traffic_flow_Preprocessing.ipynb\n",
373+
"SCOOT_df = pd.read_csv('../data/final_regression_csv/detector_530.csv')\n",
373374
"SCOOT_df_coords = SCOOT_df.loc[:,['siteId', 'lat', 'lon']]\n",
374375
"\n",
376+
"# elif you download the pre-processed traffic flow data at: Urban Big Data Centre (UBDC) and https://zenodo.org/records/12100278.\n",
377+
"SCOOT_df = pd.read_csv('../data/traffic_flow_data/locations.csv')\n",
378+
"SCOOT_df_coords = SCOOT_df.loc[:,['id', 'latitude', 'longitude']]\n",
379+
"SCOOT_df_coords = SCOOT_df_coords.rename(columns={'id': 'siteId', 'latitude': 'lat', 'longitude': 'lon'})\n",
380+
"\n",
375381
"GSV_list = []\n",
376382
"for i in range(len(SCOOT_df_coords)):\n",
377383
" lat = SCOOT_df_coords.loc[i]['lat']\n",
378384
" lon = SCOOT_df_coords.loc[i]['lon']\n",
379385
" for j in ['0', '90', '180', '270']:\n",
380-
" view = get_gsv(lat, lon, j, 'AIzaSyA21fgEbmuZMVyNJ-T92GAFFs_3meZLFVg') # your unique GSV download code\n",
386+
" view = get_gsv(lat, lon, j, key) # get your unique GSV download code https://developers.google.com/maps/documentation/streetview/overview\n",
381387
" view.save('data/' + lat + '_' + lon + '_' + j + '.jpg')\n",
382388
" \n",
383389
" GSV_download = SCOOT_df_coords.loc[[i]]\n",
@@ -549,14 +555,17 @@
549555
"metadata": {},
550556
"outputs": [],
551557
"source": [
558+
"import os\n",
552559
"new_GSV = normalisation_GSV(GSV)\n",
553-
"#new_GSV.to_csv('../Google Street View/semantic segmentation model/latest_segmentation_normalisation_output_before_2021-12-31.csv', index = False)"
560+
"save_path = '../data/independent_variables/GSV/'\n",
561+
"os.makedirs(save_path, exist_ok=True)\n",
562+
"new_GSV.to_csv(save_path + 'segmentation_normalisation_output.csv', index = False)"
554563
]
555564
}
556565
],
557566
"metadata": {
558567
"kernelspec": {
559-
"display_name": "Python 3",
568+
"display_name": "Python 3 (ipykernel)",
560569
"language": "python",
561570
"name": "python3"
562571
},
@@ -570,7 +579,7 @@
570579
"name": "python",
571580
"nbconvert_exporter": "python",
572581
"pygments_lexer": "ipython3",
573-
"version": "3.7.0"
582+
"version": "3.11.5"
574583
}
575584
},
576585
"nbformat": 4,

‎notebook/Independent_variables_processing.ipynb

+114-12
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
"outputs": [],
4444
"source": [
4545
"def read_detector_POI_join_file(detector_POI_join_shp):\n",
46-
" detector_POI_join = gpd.read_file(\"../20211028_SCOOT_Data/SCOOT_data_points-shp/Spatial Join/\" + detector_POI_join_shp + \".shp\")\n",
46+
" detector_POI_join = gpd.read_file(\"../data/independent_variables/POI/\" + detector_POI_join_shp + \".shp\")\n",
4747
" \n",
4848
" return detector_POI_join"
4949
]
@@ -136,6 +136,39 @@
136136
"## 2. Implement"
137137
]
138138
},
139+
{
140+
"cell_type": "markdown",
141+
"metadata": {},
142+
"source": [
143+
"### Unzip the file"
144+
]
145+
},
146+
{
147+
"cell_type": "code",
148+
"execution_count": 2,
149+
"metadata": {},
150+
"outputs": [],
151+
"source": [
152+
"import zipfile\n",
153+
"import os"
154+
]
155+
},
156+
{
157+
"cell_type": "code",
158+
"execution_count": 3,
159+
"metadata": {},
160+
"outputs": [],
161+
"source": [
162+
"zip_file_path = '../data/independent_variables/POI.zip'\n",
163+
"extract_dir = '../data/independent_variables/'\n",
164+
"\n",
165+
"os.makedirs(extract_dir, exist_ok=True)\n",
166+
"\n",
167+
"# Unzip the file\n",
168+
"with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:\n",
169+
" zip_ref.extractall(extract_dir)"
170+
]
171+
},
139172
{
140173
"cell_type": "markdown",
141174
"metadata": {},
@@ -165,7 +198,7 @@
165198
"all_POI.insert(loc=0, column='siteId', value=site_list)\n",
166199
"all_POI\n",
167200
"\n",
168-
"#all_POI.to_csv('all_buffer_POI_percent.csv', index = False)"
201+
"all_POI.to_csv('../data/independent_variables/POI/all_buffer_POI_percent.csv', index = False)"
169202
]
170203
},
171204
{
@@ -215,7 +248,7 @@
215248
"outputs": [],
216249
"source": [
217250
"def read_landcover_intersect_file(buffer_landcover_intersect_csv):\n",
218-
" landcover_intersect = gpd.read_file(\"../20211028_SCOOT_Data/SCOOT_data_points-shp/Spatial Join/\" + buffer_landcover_intersect_csv + \".csv\")\n",
251+
" landcover_intersect = gpd.read_file(\"../data/independent_variables/landcover/\" + buffer_landcover_intersect_csv + \".csv\")\n",
219252
" \n",
220253
" return landcover_intersect"
221254
]
@@ -272,7 +305,7 @@
272305
"source": [
273306
"def merge_landcover_class(df_landcover_percent):\n",
274307
"\n",
275-
" Landcover_osgb = gpd.read_file(\"Land_cover_shp/land_cover_osgb36.shp\")\n",
308+
" Landcover_osgb = gpd.read_file(\"../data/independent_variables/landcover/land_cover_osgb36.shp\")\n",
276309
" landcover_class = sorted(list(set(Landcover_osgb.class_2018)))\n",
277310
"\n",
278311
" urban_residential_area = [landcover_class[2],landcover_class[3], landcover_class[4], landcover_class[5], landcover_class[6], landcover_class[12]]\n",
@@ -349,6 +382,29 @@
349382
" return landcover_sta"
350383
]
351384
},
385+
{
386+
"cell_type": "markdown",
387+
"metadata": {},
388+
"source": [
389+
"### Unzip the file"
390+
]
391+
},
392+
{
393+
"cell_type": "code",
394+
"execution_count": 4,
395+
"metadata": {},
396+
"outputs": [],
397+
"source": [
398+
"zip_file_path = '../data/independent_variables/landcover.zip'\n",
399+
"extract_dir = '../data/independent_variables/'\n",
400+
"\n",
401+
"os.makedirs(extract_dir, exist_ok=True)\n",
402+
"\n",
403+
"# Unzip the file\n",
404+
"with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:\n",
405+
" zip_ref.extractall(extract_dir)"
406+
]
407+
},
352408
{
353409
"cell_type": "markdown",
354410
"metadata": {},
@@ -386,7 +442,7 @@
386442
"all_landcover.insert(loc=0, column='siteId', value=site_list)\n",
387443
"all_landcover\n",
388444
"\n",
389-
"#all_landcover.to_csv('all_buffer_landcover_percent_new.csv', index = False)"
445+
"all_landcover.to_csv('../data/independent_variables/landcover/all_buffer_landcover_percent_new.csv', index = False)"
390446
]
391447
},
392448
{
@@ -437,7 +493,7 @@
437493
"outputs": [],
438494
"source": [
439495
"def read_road_intersect_file(buffer_road_intersect_csv):\n",
440-
" road_link_intersect = gpd.read_file(\"../20211028_SCOOT_Data/SCOOT_data_points-shp/Spatial Join/\" + buffer_road_intersect_csv + \".csv\")\n",
496+
" road_link_intersect = gpd.read_file(\"../data/independent_variables/road_link/\" + buffer_road_intersect_csv + \".csv\")\n",
441497
" return road_link_intersect"
442498
]
443499
},
@@ -506,6 +562,29 @@
506562
" return road_sta"
507563
]
508564
},
565+
{
566+
"cell_type": "markdown",
567+
"metadata": {},
568+
"source": [
569+
"### Unzip the file"
570+
]
571+
},
572+
{
573+
"cell_type": "code",
574+
"execution_count": 5,
575+
"metadata": {},
576+
"outputs": [],
577+
"source": [
578+
"zip_file_path = '../data/independent_variables/road_link.zip'\n",
579+
"extract_dir = '../data/independent_variables/'\n",
580+
"\n",
581+
"os.makedirs(extract_dir, exist_ok=True)\n",
582+
"\n",
583+
"# Unzip the file\n",
584+
"with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:\n",
585+
" zip_ref.extractall(extract_dir)"
586+
]
587+
},
509588
{
510589
"cell_type": "markdown",
511590
"metadata": {},
@@ -541,7 +620,7 @@
541620
"all_road.insert(loc=0, column='siteId', value=site_list)\n",
542621
"all_road\n",
543622
"\n",
544-
"#all_road.to_csv('all_buffer_road_percent.csv', index = False)"
623+
"all_road.to_csv('../data/independent_variables/road_link/all_buffer_road_percent.csv', index = False)"
545624
]
546625
},
547626
{
@@ -670,7 +749,7 @@
670749
"outputs": [],
671750
"source": [
672751
"def read_OA_intersect_file(buffer_OA_intersect_csv):\n",
673-
" OA_link_intersect = gpd.read_file(\"../20211028_SCOOT_Data/SCOOT_data_points-shp/Spatial Join/\" + buffer_OA_intersect_csv + \".csv\")\n",
752+
" OA_link_intersect = gpd.read_file(\"../data/independent_variables/census/\" + buffer_OA_intersect_csv + \".csv\")\n",
674753
" return OA_link_intersect"
675754
]
676755
},
@@ -742,6 +821,29 @@
742821
" return census_sta"
743822
]
744823
},
824+
{
825+
"cell_type": "markdown",
826+
"metadata": {},
827+
"source": [
828+
"### Unzip the file"
829+
]
830+
},
831+
{
832+
"cell_type": "code",
833+
"execution_count": 6,
834+
"metadata": {},
835+
"outputs": [],
836+
"source": [
837+
"zip_file_path = '../data/independent_variables/census.zip'\n",
838+
"extract_dir = '../data/independent_variables/'\n",
839+
"\n",
840+
"os.makedirs(extract_dir, exist_ok=True)\n",
841+
"\n",
842+
"# Unzip the file\n",
843+
"with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:\n",
844+
" zip_ref.extractall(extract_dir)"
845+
]
846+
},
745847
{
746848
"cell_type": "markdown",
747849
"metadata": {},
@@ -765,7 +867,7 @@
765867
"doc_list = ['100', '200', '300', '400']\n",
766868
"df_list = []\n",
767869
"\n",
768-
"all_census_csv = reconstruct_census_csv('census2011_csv/useful_data/')\n",
870+
"all_census_csv = reconstruct_census_csv('../data/independent_variables/census/useful_data/')\n",
769871
"useful_census_csv_area = get_useful_census(all_census_csv)\n",
770872
"\n",
771873
"for doc in doc_list:\n",
@@ -781,7 +883,7 @@
781883
"all_census.insert(loc=0, column='siteId', value=site_list)\n",
782884
"all_census\n",
783885
"\n",
784-
"#all_census.to_csv('all_buffer_census_percent.csv', index = False)"
886+
"all_census.to_csv('../data/independent_variables/census/all_buffer_census_percent.csv', index = False)"
785887
]
786888
},
787889
{
@@ -812,7 +914,7 @@
812914
],
813915
"metadata": {
814916
"kernelspec": {
815-
"display_name": "Python 3",
917+
"display_name": "Python 3 (ipykernel)",
816918
"language": "python",
817919
"name": "python3"
818920
},
@@ -826,7 +928,7 @@
826928
"name": "python",
827929
"nbconvert_exporter": "python",
828930
"pygments_lexer": "ipython3",
829-
"version": "3.7.0"
931+
"version": "3.11.5"
830932
}
831933
},
832934
"nbformat": 4,

0 commit comments

Comments
 (0)