merged with github version

.dockerignore

@@ -0,0 +1,5 @@
+.venv
+.pytest_cache
+.idea
+__pycache__
+.vscode

Makefile

@@ -8,4 +8,4 @@ companies:
 	scrapy crawl companies -a selenium_hostname=localhost -o users.csv
 
 tests:
-	pytest linkedin/tests/*
+	pytest linkedin/tests/*

linkedin/items.py

@@ -5,3 +5,4 @@ class LinkedinUser(scrapy.Item):
     name = scrapy.Field()
     title = scrapy.Field()
     company = scrapy.Field()
+    link = scrapy.Field()

linkedin/spiders/selenium.py

@@ -120,7 +120,6 @@ def extracts_see_all_url(driver):
 
     a_elem = driver.find_element_by_link_text(see_all_ex_text)
     see_all_url = a_elem.get_attribute('href')
-
     print(f'Found the following URL: {see_all_url}')
     return see_all_url
 
@@ -141,13 +140,17 @@ def extracts_linkedin_users(driver, company):
         result = get_by_xpath_or_none(driver, last_result_xpath)
         if result is not None:
 
+
+            link_elem = get_by_xpath_or_none(result, './/*[@class="search-result__result-link ember-view"]')
+            link = link_elem.get_attribute('href') if link_elem is not None else None
+
             name_elem = get_by_xpath_or_none(result, './/*[@class="name actor-name"]')
             name = name_elem.text if name_elem is not None else None
 
             title_elem = get_by_xpath_or_none(result, './/p')
             title = title_elem.text if name_elem is not None else None
 
-            user = LinkedinUser(name=name, title=title, company=company)
+            user = LinkedinUser(name=name, title=title, company=company, link=link)
 
             yield user
 

requirements/production.txt

@@ -6,7 +6,7 @@ selenium>=3.14.0 # pyup: < 4.0  # https://github.com/SeleniumHQ/selenium
 
 # Testing
 # todo: move them to a dedicated req file
-pytest==5.3.1
+pytest==5.3.2
 pytest-sugar==0.9.2
 
 # Linkedin API library