Skip to content

Commit

Permalink
download multiple files with multiple threads in event based system u…
Browse files Browse the repository at this point in the history 
…sing JAVA 8 promises
  • Loading branch information
@harshmathur1990
harshmathur1990 committed Mar 2, 2017
0 parents commit a96ae33
Show file tree
Hide file tree
Showing 10 changed files with 601 additions and 0 deletions.
56 changes: 56 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
# Logs
logs
*.log
npm-debug.log*

# Runtime data
pids
*.pid
*.seed

# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov

# Coverage directory used by tools like istanbul
coverage

# nyc test coverage
.nyc_output

# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
.grunt

# node-waf configuration
.lock-wscript

# Compiled binary addons (http://nodejs.org/api/addons.html)
build/Release

# Dependency directories
node_modules
jspm_packages

# Optional npm cache directory
.npm

# Optional REPL history
.node_repl_history

# MacOs junk
.DS_Store

# ignore Blotter environment file
.env

# Librarian server key
librarian.key

Librarian/configs/*.json

# IntelliJ Configs
.idea/*
ThreadPoolDownload.iml

target/*

*.class
100 changes: 100 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>main.java</groupId>
<artifactId>ThreadPoolDownload</artifactId>
<version>1.0-SNAPSHOT</version>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.2.1</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.4</version>
<configuration>
<archive>
<manifestFile>src/main/resources/META-INF/MANIFEST.MF</manifestFile>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>1.6</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.download.NewMultipleDownloadManager</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.3.1</version>
</dependency>
</dependencies>
</project>
59 changes: 59 additions & 0 deletions src/main/java/com/download/ConnectionData.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package com.download;

import java.net.URL;

/**
* Created by harshmathur on 17/02/17.
*/
public class ConnectionData {

private Integer job;

private ConnectionStatus connectionStatus;

private URL url;

private final Long speed;

public String getFileName() {
return fileName;
}

private final String fileName;

public ConnectionData(Integer job, ConnectionStatus connectionStatus, URL url, Long speed, String fileName) {
this.job = job;
this.connectionStatus = connectionStatus;
this.url = url;
this.speed = speed;
this.fileName = fileName;
}

public Long getSpeed() {
return speed;
}

public Integer getJob() {
return job;
}

public void setJob(Integer job) {
this.job = job;
}

public ConnectionStatus getConnectionStatus() {
return connectionStatus;
}

public void setConnectionStatus(ConnectionStatus connectionStatus) {
this.connectionStatus = connectionStatus;
}

public URL getUrl() {
return url;
}

public void setUrl(URL url) {
this.url = url;
}
}
14 changes: 14 additions & 0 deletions src/main/java/com/download/ConnectionStatus.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package com.download;

/**
* Created by harshmathur on 17/02/17.
*/
public enum ConnectionStatus {
PART_DOWNLOADED,
PART_DOWNLOAD_FAILED,
DOWNLOADED,
DOWNLOAD_FAILED,
MERGED,
MERGE_FAILED,
ZERO_LENGTH_CONTENT
}
79 changes: 79 additions & 0 deletions src/main/java/com/download/NewDownloadWorker.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package com.download;

import com.sun.org.apache.xpath.internal.operations.Bool;
import org.apache.commons.io.FilenameUtils;

import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.util.Date;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

/**
* Created by harshmathur on 14/02/17.
*/
public class NewDownloadWorker {

private static final int BUFFER_SIZE = 256000;
private FileOutputStream out;
private final URL url;
private final Long numberofBytesPerConnection;
private final Integer connectionNumber;
private final ExecutorService service;
private final Boolean resume;

public NewDownloadWorker(Integer connectionNumber, URL url,
Long numberofBytesPerConnection, Boolean resume, ExecutorService service) {
this.url = url;
this.numberofBytesPerConnection = numberofBytesPerConnection;
this.connectionNumber = connectionNumber;
this.service = service;
this.resume = resume;
}

public ConnectionData download() {
Long startTime = new Date().getTime();
Long endTime = null;
String fileName = Utilities.getFileName(url, Integer.toString(connectionNumber), "/tmp");
try {
Long start = new Long(connectionNumber * numberofBytesPerConnection);
Long end = start+numberofBytesPerConnection-1;
if (this.resume) {
File fileExists = new File(fileName);
Long length = fileExists.length();
if (length < numberofBytesPerConnection) {
start = start + length;
} else if (length == numberofBytesPerConnection) {
return new ConnectionData(connectionNumber, ConnectionStatus.PART_DOWNLOADED, this.url, Long.MAX_VALUE, fileName);
}
}
URLConnection connection = this.url.openConnection();
connection.setRequestProperty("Range", "bytes="+start+"-"+end);
BufferedInputStream in = new BufferedInputStream(connection.getInputStream());
byte[] buffer = new byte[BUFFER_SIZE];
Integer bytesRead = -1;
this.out = new FileOutputStream(fileName);
while (true) {
bytesRead = in.read(buffer);
if(bytesRead == -1) {
break;
}
out.write(buffer, 0, bytesRead);
}
out.close();
in.close();
endTime = new Date().getTime();
return new ConnectionData(connectionNumber, ConnectionStatus.PART_DOWNLOADED, url, (end-start)/(endTime-startTime), fileName);
} catch (IOException e) {
e.printStackTrace();
return new ConnectionData(connectionNumber, ConnectionStatus.PART_DOWNLOAD_FAILED, url, 0L, null);
}
catch (Exception e) {
e.printStackTrace();
return new ConnectionData(connectionNumber, ConnectionStatus.PART_DOWNLOAD_FAILED, url, 0L, null);
}
}
}
Loading

0 comments on commit a96ae33

Please sign in to comment.