add model, update test

Author: yennanliu

dev_projects/ScrapingService/src/main/java/model/PokemonProduct.java

@@ -0,0 +1,52 @@
+package model;
+
+// https://www.zenrows.com/blog/web-scraping-java#extract-data
+
+public class PokemonProduct {
+    private String url;
+    private String image;
+    private String name;
+    private String price;
+
+    // getters and setters omitted for brevity...
+    public String getUrl() {
+        return url;
+    }
+
+    public void setUrl(String url) {
+        this.url = url;
+    }
+
+    public String getImage() {
+        return image;
+    }
+
+    public void setImage(String image) {
+        this.image = image;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getPrice() {
+        return price;
+    }
+
+    public void setPrice(String price) {
+        this.price = price;
+    }
+
+    @Override
+    public String toString() {
+        return "{ \"url\":\"" + url + "\", "
+                + " \"image\": \"" + image + "\", "
+                + "\"name\":\"" + name + "\", "
+                + "\"price\": \"" + price + "\" }";
+    }
+
+}

dev_projects/ScrapingService/src/test/java/com/yen/WebScrapingTest.java

@@ -1,5 +1,6 @@
 package com.yen;
 
+import model.PokemonProduct;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVPrinter;
 import org.jsoup.Jsoup;
@@ -17,6 +18,8 @@
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
 
 public class WebScrapingTest {
 
@@ -68,4 +71,59 @@ public void webScrapTest1() throws IOException {
 
     }
 
+
+    // https://www.zenrows.com/blog/web-scraping-java#connect-to-target-website
+    @Test
+    public void webScrapTest2(){
+
+        String URL = "https://scrapeme.live/shop";
+
+        // initializing the HTML Document page variable
+        Document doc;
+
+        List<PokemonProduct> pokemonProducts = new ArrayList<>();
+
+        try {
+            // fetching the target website
+            // doc = Jsoup.connect(URL).get();
+            doc = Jsoup
+                    .connect(URL)
+                    .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36")
+                    .header("Accept-Language", "*")
+                    .get();
+            //System.out.println("doc = " + doc);
+
+//            Elements products = doc.select("li.product");
+//            System.out.println("products = " + products);
+
+            // initializing the list of Java object to store
+            // the scraped data
+            //List<PokemonProduct> pokemonProducts = new ArrayList<>();
+
+            // retrieving the list of product HTML elements
+            Elements products = doc.select("li.product");
+
+            // iterating over the list of HTML products
+            for (Element product : products) {
+                PokemonProduct pokemonProduct = new PokemonProduct();
+
+                // extracting the data of interest from the product HTML element
+                // and storing it in pokemonProduct
+                pokemonProduct.setUrl(product.selectFirst("a").attr("href"));
+                pokemonProduct.setImage(product.selectFirst("img").attr("src"));
+                pokemonProduct.setName(product.selectFirst("h2").text());
+                pokemonProduct.setPrice(product.selectFirst("span").text());
+
+                // adding pokemonProduct to the list of the scraped products
+                pokemonProducts.add(pokemonProduct);
+            }
+
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        System.out.println(">>> pokemonProducts = ");
+        System.out.println(pokemonProducts);
+    }
+
 }