xielq 4 anni fa
parent
commit
eebbea3518
59 ha cambiato i file con 3252 aggiunte e 0 eliminazioni
  1. 16 0
      README.md
  2. 15 0
      build.gradle
  3. 12 0
      gradle/tasks.gradle
  4. BIN
      gradle/wrapper/gradle-wrapper.jar
  5. 6 0
      gradle/wrapper/gradle-wrapper.properties
  6. 172 0
      gradlew
  7. 84 0
      gradlew.bat
  8. 65 0
      oos-core/build.gradle
  9. 74 0
      oos-core/src/main/java/com/uas/platform/oos/core/common/ResultBean.java
  10. 24 0
      oos-core/src/main/java/com/uas/platform/oos/core/exception/ClientFailedException.java
  11. 23 0
      oos-core/src/main/java/com/uas/platform/oos/core/exception/MigrateFileFailedException.java
  12. 60 0
      oos-core/src/main/java/com/uas/platform/oos/core/oos/AbstractClient.java
  13. 65 0
      oos-core/src/main/java/com/uas/platform/oos/core/oos/FileOperations.java
  14. 243 0
      oos-core/src/main/java/com/uas/platform/oos/core/oos/client/OosClientHelper.java
  15. 73 0
      oos-core/src/main/java/com/uas/platform/oos/core/util/JacksonUtils.java
  16. 28 0
      oos-core/src/main/java/com/uas/platform/oos/core/validate/ArgumentValidate.java
  17. 3 0
      oos-core/src/main/resources/OOSCredentials.properties
  18. 72 0
      oos-dashboard/build.gradle
  19. 6 0
      oos-dashboard/src/main/docker/Dockerfile
  20. 26 0
      oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/DashboardApplication.java
  21. 95 0
      oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/api/FileOperationController.java
  22. 43 0
      oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/config/ControllerExceptionHandler.java
  23. 46 0
      oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/service/FileOperationService.java
  24. 54 0
      oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/service/impl/FileOperationServiceImpl.java
  25. 27 0
      oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/web/HomeController.java
  26. 3 0
      oos-dashboard/src/main/resources/OOSCredentials.properties
  27. 11 0
      oos-dashboard/src/main/resources/application-dev.yml
  28. 7 0
      oos-dashboard/src/main/resources/application-prod.yml
  29. 7 0
      oos-dashboard/src/main/resources/application-test.yml
  30. 8 0
      oos-dashboard/src/main/resources/application.yml
  31. 55 0
      oos-dashboard/src/main/resources/templates/download.html
  32. 31 0
      oos-dashboard/src/main/resources/templates/index.html
  33. 142 0
      oos-dashboard/src/main/resources/templates/upload.html
  34. 48 0
      oos-migration/build.gradle
  35. 6 0
      oos-migration/src/main/docker/Dockerfile
  36. 26 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/MigrationApplication.java
  37. 62 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/api/MigrationController.java
  38. 43 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/config/ControllerExceptionHandler.java
  39. 23 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/dao/MigrationFileRepository.java
  40. 12 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/dao/MockUrlRepository.java
  41. 29 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/dao/ResourceFieldsRepository.java
  42. 118 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/entity/MigrationFile.java
  43. 67 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/entity/MockUrl.java
  44. 37 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/entity/OperationResult.java
  45. 99 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/entity/ResourceFields.java
  46. 51 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/service/MigrationFileService.java
  47. 10 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/service/MockUrlService.java
  48. 452 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/service/impl/MigrationFileServiceImpl.java
  49. 43 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/service/impl/MockUrlServiceImpl.java
  50. 31 0
      oos-migration/src/main/java/com/uas/platform/oos/migration/web/HomeController.java
  51. 3 0
      oos-migration/src/main/resources/OOSCredentials.properties
  52. 22 0
      oos-migration/src/main/resources/application-dev.yml
  53. 22 0
      oos-migration/src/main/resources/application-test.yml
  54. 12 0
      oos-migration/src/main/resources/application.yml
  55. 32 0
      oos-migration/src/main/resources/templates/index.html
  56. 135 0
      oos-migration/src/main/resources/templates/migration/create-record.html
  57. 132 0
      oos-migration/src/main/resources/templates/migration/move-file.html
  58. 137 0
      oos-migration/src/main/resources/templates/migration/sync-new-url.html
  59. 4 0
      settings.gradle

+ 16 - 0
README.md

@@ -0,0 +1,16 @@
+# README
+
+## Introduction
+
+OOS service supplies file upload and download functions.
+
+## How to use
+
+1. Login this  [OOS website](http://oos-gz.ctyun.cn).
+2. Create one bucket, make it public-read; Otherwise, can't read the object by url.
+3. Create a key for SDK client visiting this resource pool
+4. Configure the bucket and key
+5. Bootstrap OOS service
+
+> Note that guangzhou OOS endpoint is `https://oos-gz.ctyunapi.cn` , console website is `https://oos-gz.ctyun.cn` , resource url pattern is `https://oos-gz.ctyunapi.cn/<Bucket Name>/<Object Key>`
+

+ 15 - 0
build.gradle

@@ -0,0 +1,15 @@
+group 'com.uas.platform.oos'
+version '1.0.1-SNAPSHOT'
+
+subprojects {
+    apply plugin: 'java'
+
+    sourceCompatibility = 1.8
+
+    repositories {
+        mavenLocal()
+        maven { url 'http://10.10.101.21:8081/artifactory/ext-release-local' }
+        jcenter()
+        mavenCentral()
+    }
+}

+ 12 - 0
gradle/tasks.gradle

@@ -0,0 +1,12 @@
+// Gradle Tasks Configurations
+// Created by huxz on 2017-3-17 14:39:36
+bootRun {
+	addResources = true
+}
+
+docker {
+	name "${dcokerRegistry}/${project.name}:${project.version}"
+	tags "latest", "dev"
+	dockerfile "${projectDir}/src/main/docker/Dockerfile"
+	files "${buildDir}/libs/${project.name}.jar"
+}.dependsOn build

BIN
gradle/wrapper/gradle-wrapper.jar


+ 6 - 0
gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Fri Sep 22 14:01:10 CST 2017
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-bin.zip

+ 172 - 0
gradlew

@@ -0,0 +1,172 @@
+#!/usr/bin/env sh
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn ( ) {
+    echo "$*"
+}
+
+die ( ) {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save ( ) {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"

+ 84 - 0
gradlew.bat

@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

+ 65 - 0
oos-core/build.gradle

@@ -0,0 +1,65 @@
+group 'com.uas.platform.oos'
+version '1.0.1-SNAPSHOT'
+
+apply plugin: 'maven-publish'
+
+dependencies {
+    //Apache Commons
+    compile ("org.apache.commons:commons-lang3:3.5")
+    compile ("commons-io:commons-io:2.5")
+    compile "commons-logging:commons-logging:1.2"
+
+    compile "com.fasterxml.jackson.core:jackson-core:2.8.1"
+    compile "com.fasterxml.jackson.core:jackson-databind:2.8.1"
+
+    compile 'com.amazonaws:aws-java-sdk:1.4.7'
+    compile "org.springframework:spring-web:4.3.11.RELEASE"
+
+}
+
+javadoc {
+    options {
+        encoding "UTF-8"
+        charSet "UTF-8"
+        author true
+        version true
+        links "http://docs.oracle.com/javase/8/docs/api"
+    }
+}
+
+// Package sources
+task sourceJar(type: Jar) {
+    classifier "sources"
+    from sourceSets.main.allJava
+}
+
+// Package javadoc
+task docJar(type: Jar, dependsOn: javadoc) {
+    classifier "javadoc"
+    from javadoc.destinationDir
+}
+
+publishing {
+    publications {
+        // Publish maven snapshot
+        coreSnapshot(MavenPublication) {
+            groupId project.group
+            artifactId project.name
+            version project.version
+
+            from components.java
+            artifact sourceJar
+            artifact docJar
+        }
+    }
+
+    repositories {
+        maven {
+            credentials {
+                username "yingp"
+                password "111111"
+            }
+            url "http://113.105.74.141:8081/artifactory/libs-snapshot-local"
+        }
+    }
+}

+ 74 - 0
oos-core/src/main/java/com/uas/platform/oos/core/common/ResultBean.java

@@ -0,0 +1,74 @@
+package com.uas.platform.oos.core.common;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import java.io.Serializable;
+
+@SuppressWarnings("unused")
+@JsonInclude(Include.NON_EMPTY)
+public class ResultBean<T> implements Serializable {
+
+  private static final long serialVersionUID = -8666185459443067060L;
+
+  private Boolean success;
+
+  private T data;
+
+  private ErrorCode code;
+
+  private String message;
+
+  enum ErrorCode {
+    FAIL, NO_PERMISSION
+  }
+
+  public ResultBean(T data) {
+    this.success = true;
+    this.data = data;
+  }
+
+  /**
+   * response result when error occurs.
+   *
+   * @param e   exception info
+   */
+  public ResultBean(Throwable e) {
+    this.success = false;
+    this.message = e.getMessage();
+
+    // Set default code
+    this.code = ErrorCode.FAIL;
+  }
+
+  public Boolean getSuccess() {
+    return success;
+  }
+
+  public void setSuccess(Boolean success) {
+    this.success = success;
+  }
+
+  public T getData() {
+    return data;
+  }
+
+  public void setData(T data) {
+    this.data = data;
+  }
+
+  public ErrorCode getCode() {
+    return code;
+  }
+
+  public void setCode(ErrorCode code) {
+    this.code = code;
+  }
+
+  public String getMessage() {
+    return message;
+  }
+
+  public void setMessage(String message) {
+    this.message = message;
+  }
+}

+ 24 - 0
oos-core/src/main/java/com/uas/platform/oos/core/exception/ClientFailedException.java

@@ -0,0 +1,24 @@
+package com.uas.platform.oos.core.exception;
+
+/**
+ * OSS SDK client execution failed when making request, or handling
+ * the response, or server error.
+ *
+ * @author huxz
+ */
+public class ClientFailedException extends RuntimeException {
+
+  private static final long serialVersionUID = 4610788846347847922L;
+
+  public ClientFailedException(String message) {
+    super(message);
+  }
+
+  public ClientFailedException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public ClientFailedException(Throwable cause) {
+    super(cause);
+  }
+}

+ 23 - 0
oos-core/src/main/java/com/uas/platform/oos/core/exception/MigrateFileFailedException.java

@@ -0,0 +1,23 @@
+package com.uas.platform.oos.core.exception;
+
+/**
+ * Throw exceptions when migrate remote file to OOS.
+ *
+ * @author huxz
+ */
+public class MigrateFileFailedException extends RuntimeException {
+
+  private static final long serialVersionUID = -2603692927730915466L;
+
+  public MigrateFileFailedException(String message) {
+    super(message);
+  }
+
+  public MigrateFileFailedException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public MigrateFileFailedException(Throwable cause) {
+    super(cause);
+  }
+}

+ 60 - 0
oos-core/src/main/java/com/uas/platform/oos/core/oos/AbstractClient.java

@@ -0,0 +1,60 @@
+package com.uas.platform.oos.core.oos;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.PropertiesCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.uas.platform.oos.core.exception.ClientFailedException;
+import com.uas.platform.oos.core.oos.client.OosClientHelper;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Abstract class provides method to create AmazonS3 instance.
+ *
+ * @author huxz
+ */
+public abstract class AbstractClient {
+
+  /**
+   * OOS endpoint.
+   */
+  public static final String OOS_ENDPOINT = "oos-gz.ctyunapi.cn";
+
+  /**
+   * OOS credentials properties file name.
+   */
+  private static final String CREDENTIALS_PROP_NAME = "OOSCredentials.properties";
+
+  /**
+   * Default Class Path of OOS credential properties.
+   */
+  private static final String CREDENTIALS_CLASS_PATH = "/" + CREDENTIALS_PROP_NAME;
+
+  /**
+   * Create a AmazonS3 instance, and configure it using default constants.
+   */
+  protected static AmazonS3 getClient() {
+
+    // Load OOS credentials from class path.
+    InputStream stream = OosClientHelper.class.getResourceAsStream(CREDENTIALS_CLASS_PATH);
+    if (stream == null) {
+      throw new IllegalStateException("Can't find properties " + CREDENTIALS_PROP_NAME);
+    }
+
+    // Create ClientConfiguration instance for configuring retry.
+    ClientConfiguration configuration = new ClientConfiguration();
+    configuration.setMaxErrorRetry(3);
+    configuration.setConnectionTimeout(30 * 1000);
+    configuration.setSocketTimeout(30 * 1000);
+
+    AmazonS3 client;
+    try {
+      client = new AmazonS3Client(new PropertiesCredentials(stream), configuration);
+      client.setEndpoint("https://" + OOS_ENDPOINT);
+    } catch (IOException e) {
+      throw new ClientFailedException("Failed to create AmazonS3 client ", e);
+    }
+    return client;
+  }
+}

+ 65 - 0
oos-core/src/main/java/com/uas/platform/oos/core/oos/FileOperations.java

@@ -0,0 +1,65 @@
+package com.uas.platform.oos.core.oos;
+
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import java.io.File;
+import java.util.Map;
+import org.springframework.web.multipart.MultipartFile;
+
+/**
+ * Abstract interface defines basic operations of file.
+ *
+ * @author huxz
+ */
+public interface FileOperations {
+
+  /**
+   * Upload local file to OOS.
+   *
+   * @param bucketName  bucket name
+   * @param file  local file to be uploaded
+   * @return  path of file uploaded
+   */
+  String upload(String bucketName, File file);
+
+  /**
+   * Upload file from data of form to OOS.
+   *
+   * @param bucketName  bucket name
+   * @param multiFile file inside form data
+   * @return  path of file uploaded
+   */
+  String upload(String bucketName, MultipartFile multiFile);
+
+  /**
+   * Upload remote file to OOS.
+   *
+   * @param bucketName  bucket name
+   * @param url link of remote file
+   * @return  path of file uploaded
+   */
+  String upload(String bucketName, String url);
+
+  /**
+   * Download file from OOS.
+   *
+   * @param url the url of file on OOS
+   * @return  InputStream of remote file
+   */
+  S3ObjectInputStream download(String url);
+
+  /**
+   * Delete file on OOS.
+   *
+   * @param url the url of file on OOS
+   */
+  void delete(String url);
+
+  /**
+   * Gain file metadata from OOS.
+   *
+   * @param url the url of file on OOS
+   * @return metadata of the file
+   */
+  Map<String, Object> showMetadata(String url);
+
+}

+ 243 - 0
oos-core/src/main/java/com/uas/platform/oos/core/oos/client/OosClientHelper.java

@@ -0,0 +1,243 @@
+package com.uas.platform.oos.core.oos.client;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.uas.platform.oos.core.oos.AbstractClient;
+import com.uas.platform.oos.core.oos.FileOperations;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Map;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.Validate;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.util.StringUtils;
+import org.springframework.web.multipart.MultipartFile;
+
+/**
+ * A helper class provides OOS objects operations.
+ *
+ * @author huxz
+ */
+public class OosClientHelper extends AbstractClient implements FileOperations {
+
+  private static final Log logger = LogFactory.getLog(OosClientHelper.class);
+
+  /**
+   * A bucket name used to test.
+   */
+  private static final String BUCKET_FOR_TEST = "uas.test";
+
+  private static final String GROUP_TAG_NAME = "group";
+
+  @Override
+  public String upload(String bucketName, File file) {
+    Validate.notNull(file, "Uploading file can't be Null");
+    if (StringUtils.isEmpty(bucketName)) {
+      bucketName = BUCKET_FOR_TEST;
+    }
+
+    AmazonS3 client = getClient();
+
+    // Check whether bucket exists
+    if (!client.doesBucketExist(bucketName)) {
+      throw new IllegalArgumentException(String.format("Bucket {%s} doesn't exist", bucketName));
+    }
+
+    String key = generateObjectKey(file.getName());
+    client.putObject(bucketName, key, file);
+
+    return spliceResourceUrl(bucketName, key);
+  }
+
+  @Override
+  public String upload(String bucketName, MultipartFile multiFile) {
+    Validate.notNull(multiFile, "Uploading file can't be Null");
+    if (StringUtils.isEmpty(bucketName)) {
+      bucketName = BUCKET_FOR_TEST;
+    }
+
+    AmazonS3 client = getClient();
+
+    // Check whether bucket exists
+    if (!client.doesBucketExist(bucketName)) {
+      throw new IllegalArgumentException(String.format("Bucket {%s} doesn't exist", bucketName));
+    }
+
+
+    final String key = generateObjectKey(multiFile.getOriginalFilename());
+
+    ObjectMetadata metadata = new ObjectMetadata();
+    metadata.setContentLength(multiFile.getSize());
+    metadata.setContentType(multiFile.getContentType());
+
+    try {
+      client.putObject(bucketName, key, multiFile.getInputStream(), metadata);
+
+      return spliceResourceUrl(bucketName, key);
+    } catch (IOException e) {
+      throw new IllegalStateException("Read file failed", e);
+    }
+  }
+
+  @Override
+  public String upload(String bucketName, String url) {
+    Validate.notBlank(url, "Remote file url must be non-empty");
+    if (StringUtils.isEmpty(bucketName)) {
+      bucketName = BUCKET_FOR_TEST;
+    }
+
+    AmazonS3 client = getClient();
+
+    // Check whether bucket exists
+    if (!client.doesBucketExist(bucketName)) {
+      throw new IllegalArgumentException(String.format("Bucket {%s} doesn't exist", bucketName));
+    }
+
+    try {
+      URL resource = new URL(url);
+      HttpURLConnection connection = (HttpURLConnection) resource.openConnection();
+      connection.setReadTimeout(30 * 1000);
+      connection.connect();
+
+      String extension = FilenameUtils.getExtension(url);
+
+      // Cache remote file to local before upload to OOS
+      File tempFile = File.createTempFile("oos-remote", "." + extension);
+      FileOutputStream fos = new FileOutputStream(tempFile);
+      BufferedInputStream bis = new BufferedInputStream(connection.getInputStream());
+
+      byte[] buf = new byte[1024];
+      int len;
+      while ((len = bis.read(buf)) >= 0) {
+        fos.write(buf, 0 ,len);
+      }
+      fos.close();
+      bis.close();
+
+      String filePath = upload(bucketName, tempFile);
+      //noinspection ResultOfMethodCallIgnored
+      tempFile.delete();
+
+      return filePath;
+    } catch (IOException e) {
+      throw new IllegalStateException("Read file failed", e);
+    }
+  }
+
+  @Override
+  public S3ObjectInputStream download(String url) {
+    validateUrl(url);
+
+    FileInfo info = new FileInfo(url).getFileInfo();
+
+    S3Object s3Object = getClient().getObject(info.bucketName, info.key);
+    if (s3Object == null) {
+      throw new IllegalStateException("File doesn't exist");
+    }
+    return s3Object.getObjectContent();
+  }
+
+  @Override
+  public void delete(String url) {
+    validateUrl(url);
+
+    FileInfo info = new FileInfo(url).getFileInfo();
+
+    getClient().deleteObject(info.bucketName, info.key);
+  }
+
+  @Override
+  public Map<String, Object> showMetadata(String url) {
+    validateUrl(url);
+
+    FileInfo info = new FileInfo(url).getFileInfo();
+
+    ObjectMetadata metadata = getClient().getObjectMetadata(info.bucketName, info.key);
+    if (metadata == null) {
+      throw new IllegalStateException("File doesn't exist");
+    }
+    return metadata.getRawMetadata();
+  }
+
+  /**
+   * Validate url which is used to download, delete or get metadata.
+   *
+   * @param url link of remote file
+   */
+  private void validateUrl(String url) {
+    Validate.notBlank(url, "File url must be non-empty");
+    if (!url.contains(OOS_ENDPOINT)) {
+      throw new IllegalArgumentException("This url doesn't belong to OOS");
+    }
+  }
+
+  /**
+   * Generate object key for files to be uploaded.
+   *
+   * @param filename  the name of file uploading
+   * @return  object key generated by random string
+   */
+  private String generateObjectKey(String filename) {
+    final String extension = FilenameUtils.getExtension(filename);
+
+    String[] randomStrs = new String[4];
+    randomStrs[0] = GROUP_TAG_NAME + RandomStringUtils.randomNumeric(1);
+    randomStrs[1] = RandomStringUtils.randomAlphabetic(1).toUpperCase() +
+        RandomStringUtils.randomNumeric(2);
+    randomStrs[2] = RandomStringUtils.randomAlphanumeric(2);
+    randomStrs[3] = RandomStringUtils.randomAlphanumeric(2);
+    final String prefix = String.join("/", randomStrs);
+
+    final String baseName = "CgpkyFk" + RandomStringUtils.randomAlphanumeric(24);
+
+    return prefix + "/" + baseName + "." + extension;
+  }
+
+  /**
+   * Gain resource url when file upload successfully.
+   *
+   * @param bucketName  the name of bucket
+   * @param key the key of Object
+   * @return  resource url
+   */
+  private String spliceResourceUrl(String bucketName, String key) {
+    return "http://" + bucketName + "." +
+        OOS_ENDPOINT + "/" +
+        key;
+  }
+
+  /**
+   * Method class gains bucket and key from url.
+   */
+  private class FileInfo {
+
+    private String url;
+    private String bucketName;
+    private String key;
+
+    FileInfo(String url) {
+      this.url = url;
+    }
+
+    FileInfo getFileInfo() {
+      String pathUrl = url.replaceFirst("http://", "")
+          .replaceFirst("." + OOS_ENDPOINT, "");
+      int index = pathUrl.indexOf("/");
+      bucketName = pathUrl.substring(0, index);
+      key = pathUrl.substring(index + 1);
+
+      if (logger.isDebugEnabled()) {
+        logger.debug("BucketName: " + bucketName + ", Key: " + key);
+      }
+      return this;
+    }
+  }
+}

+ 73 - 0
oos-core/src/main/java/com/uas/platform/oos/core/util/JacksonUtils.java

@@ -0,0 +1,73 @@
+package com.uas.platform.oos.core.util;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Jackson Util class.
+ *
+ * @author huxz
+ */
+@SuppressWarnings("unused")
+public final class JacksonUtils {
+
+  private static final Log logger = LogFactory.getLog(JacksonUtils.class);
+
+  private static final String EMPTY_JSON = "{}";
+
+  private static ObjectMapper objectMapper;
+
+  /**
+   * 把JSON文本parse为JavaBean.
+   */
+  public static <T> T fromJson(String text, Class<T> clazz) {
+    if (objectMapper == null) {
+      objectMapper = new ObjectMapper();
+    }
+    try {
+      return objectMapper.readValue(text, clazz);
+    } catch (IOException e) {
+      logger.error("Read value failed", e);
+    }
+    return null;
+  }
+
+  /**
+   * 把JSON文本parse成JavaBean集合.
+   */
+  public static <T> List<T> fromJsonArray(String text, Class<T> clazz) {
+    if (objectMapper == null) {
+      objectMapper = new ObjectMapper();
+    }
+    JavaType javaType = objectMapper.getTypeFactory()
+        .constructParametricType(ArrayList.class, clazz);
+    try {
+      return objectMapper.readValue(text, javaType);
+    } catch (IOException e) {
+      logger.error("Read value failed", e);
+    }
+    return Collections.emptyList();
+  }
+
+  /**
+   * 将JavaBean序列化为JSON文本.
+   */
+  public static String toJson(Object object) {
+    if (objectMapper == null) {
+      objectMapper = new ObjectMapper();
+    }
+    try {
+      return objectMapper.writeValueAsString(object);
+    } catch (JsonProcessingException e) {
+      logger.error("Process json string failed", e);
+    }
+    return EMPTY_JSON;
+  }
+}

+ 28 - 0
oos-core/src/main/java/com/uas/platform/oos/core/validate/ArgumentValidate.java

@@ -0,0 +1,28 @@
+package com.uas.platform.oos.core.validate;
+
+public class ArgumentValidate {
+
+  /**
+   * Assert parameter is not null.
+   *
+   * @param parameterValue    the value of parameter
+   * @param errorMessage      the error message
+   */
+  public static void assertParameterNotNull(Object parameterValue, String errorMessage) {
+    if (parameterValue == null) {
+      throw new IllegalArgumentException(errorMessage);
+    }
+  }
+
+  /**
+   * Assert string type parameter is not null or empty.
+   *
+   * @param parameterValue    the value of parameter
+   * @param errorMessage      the error message
+   */
+  public static void assertStringNotEmpty(String parameterValue, String errorMessage) {
+    if (parameterValue == null || parameterValue.length() == 0) {
+      throw new IllegalArgumentException(errorMessage);
+    }
+  }
+}

+ 3 - 0
oos-core/src/main/resources/OOSCredentials.properties

@@ -0,0 +1,3 @@
+## OOS api key
+accessKey=4d39248aaa33fab47363
+secretKey=344b89df09e717ba29f6487f6f472438a7a7e217

+ 72 - 0
oos-dashboard/build.gradle

@@ -0,0 +1,72 @@
+buildscript {
+    ext {
+        springBootVersion = '1.5.7.RELEASE'
+        dockerVersion = '0.12.0'
+        dcokerRegistry = "10.10.100.200:5000"
+    }
+    repositories {
+        mavenLocal()
+        maven { url "https://plugins.gradle.org/m2/" }
+        maven { url "http://maven.aliyun.com/nexus/content/groups/public/" }
+        maven { url 'http://10.10.101.21:8081/artifactory/ext-release-local' }
+        jcenter()
+        mavenCentral()
+    }
+    dependencies {
+        classpath "org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}"
+        classpath "gradle.plugin.com.palantir.gradle.docker:gradle-docker:${dockerVersion}"
+    }
+}
+
+group 'com.uas.platform.oos'
+version '1.0.3'
+
+apply plugin: "com.palantir.docker"
+apply plugin: 'org.springframework.boot'
+apply plugin: 'maven-publish'
+
+apply from: "$rootDir/gradle/tasks.gradle"
+
+dependencies {
+    compile project(":oos-core")
+    compile "org.springframework.boot:spring-boot-starter-web"
+    compile "org.springframework.boot:spring-boot-starter-thymeleaf"
+
+    compile "org.webjars:webjars-locator"
+    compile "org.webjars:jquery:2.1.1"
+    compile "org.webjars:angularjs:1.4.3"
+    compile "org.webjars:bootstrap:3.2.0"
+
+    testCompile "org.springframework.boot:spring-boot-starter-test"
+}
+
+jar {
+    baseName = project.name
+    version = ''
+}
+
+publishing {
+    publications {
+        // Publish maven snapshot
+        coreSnapshot(MavenPublication) {
+            groupId project.group
+            artifactId project.name
+            version project.version
+
+            artifact source: "${buildDir}/libs/${project.name}.jar", extension: 'jar'
+            // artifact "${buildDir}/libs/${project.name}.jar"
+        }
+    }
+
+    repositories {
+        maven {
+            credentials {
+                username "yingp"
+                password "111111"
+            }
+            url "http://10.10.101.21:8081/artifactory/libs-release-local"
+        }
+    }
+}
+
+build.finalizedBy(publish, publishToMavenLocal)

+ 6 - 0
oos-dashboard/src/main/docker/Dockerfile

@@ -0,0 +1,6 @@
+FROM hub.c.163.com/library/java:8-jre-alpine
+VOLUME /tmp
+ADD oos-dashboard.jar app.jar
+RUN sh -c "touch /app.jar"
+ENV JAVA_OPTS=""
+ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar /app.jar"]

+ 26 - 0
oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/DashboardApplication.java

@@ -0,0 +1,26 @@
+package com.uas.platform.oos.dashboard;
+
+import org.springframework.boot.Banner.Mode;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+
+/**
+ * Bootstrap dashboard application.
+ *
+ * @author huxz
+ */
+@SpringBootApplication
+public class DashboardApplication {
+
+  /**
+   * Application entry.
+   *
+   * @param args    command line parameters
+   */
+  public static void main(String[] args) {
+    new SpringApplicationBuilder(DashboardApplication.class)
+        .bannerMode(Mode.OFF)
+        .web(true)
+        .run(args);
+  }
+}

+ 95 - 0
oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/api/FileOperationController.java

@@ -0,0 +1,95 @@
+package com.uas.platform.oos.dashboard.api;
+
+import com.uas.platform.oos.core.common.ResultBean;
+import com.uas.platform.oos.dashboard.service.FileOperationService;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.util.FileCopyUtils;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+
+/**
+ * Http api interface of file operations.
+ *
+ * @author huxz
+ */
+@RestController
+@RequestMapping(value = "/file")
+public class FileOperationController {
+
+  private final FileOperationService operationService;
+
+  @Autowired
+  public FileOperationController(
+      FileOperationService operationService) {
+    this.operationService = operationService;
+  }
+
+  /**
+   * Upload file to OOS.
+   *
+   * @param file  an uploaded file
+   * @param bucketName  bucket name
+   * @return  http://<Bucket-Name>.oos-gz.ctyunapi.cn/groupX/XXX/XX/XX/XX...XX.jpg
+   */
+  @PostMapping(value = "/upload", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
+  public ResultBean<String> uploadFile(@RequestParam(name = "file") MultipartFile file,
+      @RequestParam(defaultValue = "uas.test") String bucketName) {
+    return new ResultBean<>(operationService.uploadFileToRemote(bucketName, file));
+  }
+
+  /**
+   * Upload remote file to OOS.
+   *
+   * @param url an url of remote file
+   * @param bucketName  bucket name
+   * @return  http://<Bucket-Name>.oos-gz.ctyunapi.cn/groupX/XXX/XX/XX/XX...XX.jpg
+   */
+  @PostMapping(value = "/upload-remote", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
+  public ResultBean<String> uploadRemoteFile(@RequestParam String url,
+      @RequestParam(defaultValue = "uas.test") String bucketName) {
+    return new ResultBean<>(operationService.uploadRemoteFileToOos(bucketName, url));
+  }
+
+  /**
+   * Download an uploaded file.
+   *
+   * @param url an url of uploaded file
+   * @return  file output stream
+   */
+  @GetMapping(value = "/download")
+  public ResponseEntity<byte[]> downloadFile(@RequestParam String url) throws IOException {
+
+    try (BufferedInputStream bis = new BufferedInputStream(operationService.download(url))) {
+      String fileName = url.substring(url.lastIndexOf("/") + 1);
+
+      HttpHeaders headers = new HttpHeaders();
+      headers.setContentDispositionFormData("attachment", fileName);
+      headers.setContentType(MediaType.APPLICATION_OCTET_STREAM);
+
+      // Copy input stream to byte array, and create response entity
+      return new ResponseEntity<>(FileCopyUtils.copyToByteArray(bis), headers, HttpStatus.CREATED);
+    }
+  }
+
+  /**
+   * Delete an uploaded file from OOS.
+   *
+   * @param url an url of uploaded file
+   * @return  true, if operation succeed
+   */
+  @DeleteMapping(value = "/delete", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
+  public ResultBean<Boolean> removeFile(@RequestParam String url) {
+    return new ResultBean<>(operationService.delete(url));
+  }
+}

+ 43 - 0
oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/config/ControllerExceptionHandler.java

@@ -0,0 +1,43 @@
+package com.uas.platform.oos.dashboard.config;
+
+import com.uas.platform.oos.core.common.ResultBean;
+import java.io.IOException;
+import org.apache.log4j.Logger;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.http.converter.HttpMessageConverter;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
+
+/**
+ * Use @ControllerAdvice annotation to handler exceptions.
+ *
+ * <p>Tip: ResponseEntityExceptionHandler, write to the response with a {@link HttpMessageConverter
+ * message converter}</p>
+ *
+ * @author huxz
+ */
+@ControllerAdvice(annotations = {RestController.class})
+public class ControllerExceptionHandler extends ResponseEntityExceptionHandler {
+
+  private static final Logger logger = Logger.getLogger(ControllerExceptionHandler.class);
+
+  /**
+   * Handler IOException and RuntimeException.
+   *
+   * @param ex exception info
+   */
+  @ExceptionHandler(value = {IOException.class, RuntimeException.class})
+  @ResponseBody
+  public ResponseEntity<?> handlerCommonExceptions(Throwable ex) {
+    logger.error("System Error occurs", ex);
+    HttpHeaders headers = new HttpHeaders();
+    headers.add("Content-Type", MediaType.APPLICATION_JSON_UTF8_VALUE);
+    return new ResponseEntity<>(new ResultBean<>(ex), headers, HttpStatus.INTERNAL_SERVER_ERROR);
+  }
+}

+ 46 - 0
oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/service/FileOperationService.java

@@ -0,0 +1,46 @@
+package com.uas.platform.oos.dashboard.service;
+
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import org.springframework.web.multipart.MultipartFile;
+
+/**
+ * A service of uploading file or downloading files.
+ *
+ * @author huxz
+ */
+public interface FileOperationService {
+
+  /**
+   * Upload a file to OOS.
+   *
+   * @param bucketName    the name of bucket
+   * @param file          the uploaded file
+   * @return path of the file
+   */
+  String uploadFileToRemote(String bucketName, MultipartFile file);
+
+  /**
+   * Upload a remote file to OOS.
+   *
+   * @param bucketName    the name of bucket
+   * @param url           the remote file url
+   * @return path of this url
+   */
+  String uploadRemoteFileToOos(String bucketName, String url);
+
+  /**
+   * Download files from OOS.
+   *
+   * @param url link of file on OOS
+   * @return file input steam
+   */
+  S3ObjectInputStream download(String url);
+
+  /**
+   * Delete file on OOS.
+   *
+   * @param url link of file on OOS
+   * @return  true, if success
+   */
+  boolean delete(String url);
+}

+ 54 - 0
oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/service/impl/FileOperationServiceImpl.java

@@ -0,0 +1,54 @@
+package com.uas.platform.oos.dashboard.service.impl;
+
+import static com.uas.platform.oos.core.validate.ArgumentValidate.assertParameterNotNull;
+import static com.uas.platform.oos.core.validate.ArgumentValidate.assertStringNotEmpty;
+
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.uas.platform.oos.core.oos.client.OosClientHelper;
+import com.uas.platform.oos.dashboard.service.FileOperationService;
+import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
+
+/**
+ * Basic Java Bean implementation of {@code FileOperationService}.
+ *
+ * @author huxz
+ */
+@Service
+public class FileOperationServiceImpl implements FileOperationService {
+
+  @Override
+  public String uploadFileToRemote(String bucketName, MultipartFile file) {
+    assertStringNotEmpty(bucketName, "Bucket name must be non-empty");
+    assertParameterNotNull(file, "Can't transfer an empty file to OOS");
+
+    OosClientHelper helper = new OosClientHelper();
+    return helper.upload(bucketName, file);
+  }
+
+  @Override
+  public String uploadRemoteFileToOos(String bucketName, String url) {
+    assertStringNotEmpty(bucketName, "Bucket name must be non-empty");
+    assertStringNotEmpty(url, "Remote file url must be non-empty");
+
+    OosClientHelper helper = new OosClientHelper();
+    return helper.upload(bucketName, url);
+  }
+
+  @Override
+  public S3ObjectInputStream download(String url) {
+    assertStringNotEmpty(url, "Remote file url must be non-empty");
+
+    OosClientHelper helper = new OosClientHelper();
+    return helper.download(url);
+  }
+
+  @Override
+  public boolean delete(String url) {
+    assertStringNotEmpty(url, "Remote file url must be non-empty");
+
+    OosClientHelper helper = new OosClientHelper();
+    helper.delete(url);
+    return true;
+  }
+}

+ 27 - 0
oos-dashboard/src/main/java/com/uas/platform/oos/dashboard/web/HomeController.java

@@ -0,0 +1,27 @@
+package com.uas.platform.oos.dashboard.web;
+
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+
+@Controller
+@RequestMapping(value = "/")
+public class HomeController {
+
+  @RequestMapping(method = RequestMethod.GET, value = {"index", ""})
+  public String home() {
+    return "index";
+  }
+
+  @GetMapping(value = "download")
+  public String download() {
+    return "download";
+  }
+
+  @GetMapping(value = "upload")
+  public String upload() {
+    return "upload";
+  }
+
+}

+ 3 - 0
oos-dashboard/src/main/resources/OOSCredentials.properties

@@ -0,0 +1,3 @@
+## OOS api key
+accessKey=4d39248aaa33fab47363
+secretKey=344b89df09e717ba29f6487f6f472438a7a7e217

+ 11 - 0
oos-dashboard/src/main/resources/application-dev.yml

@@ -0,0 +1,11 @@
+
+spring:
+  ####
+  # Template Engine
+  ##
+  thymeleaf:
+    cache: false
+
+logging:
+  level:
+    com.uas.platform.oos: debug

+ 7 - 0
oos-dashboard/src/main/resources/application-prod.yml

@@ -0,0 +1,7 @@
+
+spring:
+  ####
+  # Template Engine
+  ##
+  thymeleaf:
+    cache: false

+ 7 - 0
oos-dashboard/src/main/resources/application-test.yml

@@ -0,0 +1,7 @@
+
+spring:
+  ####
+  # Template Engine
+  ##
+  thymeleaf:
+    cache: false

+ 8 - 0
oos-dashboard/src/main/resources/application.yml

@@ -0,0 +1,8 @@
+server:
+  port: 20290
+
+spring:
+  application:
+    name: oos-dashboard
+  profiles:
+    active: prod

+ 55 - 0
oos-dashboard/src/main/resources/templates/download.html

@@ -0,0 +1,55 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8"/>
+  <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
+  <title>Download(Dashboard)</title>
+  <meta name="description" content=""/>
+  <meta name="viewport" content="width=device-width"/>
+  <base href="/"/>
+  <link rel="stylesheet" type="text/css"
+        href="/webjars/bootstrap/css/bootstrap.min.css"/>
+</head>
+<body>
+<div class="container" ng-app="app" ng-controller="DownloadController as download">
+  <h1>文件下载
+    <small><a href="index">首页</a></small>
+  </h1>
+  <div class="row" style="min-height: 300px;">
+    <!-- TODO download file test -->
+    <div class="col-sm-6">
+      <form>
+        <div class="form-group" style="margin-top: 20px; margin-bottom: 20px;">
+          <label for="resourceUrl">远程文件下载</label>
+          <input type="url" class="form-control" id="resourceUrl" name="url" placeholder="URL"
+                 ng-model="download.resourceUrl"/>
+        </div>
+        <button class="btn btn-default" ng-click="download.downloadResource()">下载</button>
+      </form>
+    </div>
+  </div>
+  <div>
+    <a href="upload">文件上传</a>
+  </div>
+</div>
+
+<script type="text/javascript" src="/webjars/jquery/jquery.min.js"></script>
+<script type="text/javascript" src="/webjars/bootstrap/js/bootstrap.min.js"></script>
+<script type="text/javascript" src="/webjars/angularjs/angular.min.js"></script>
+<script type="text/javascript">
+  angular.module("app", [])
+  .config(function ($httpProvider) {
+    $httpProvider.defaults.headers.common['X-Requested-With'] = 'XMLHttpRequest';
+  }).controller("DownloadController", function () {
+    const self = this;
+
+    self.resourceUrl = '';
+
+    self.downloadResource = function () {
+      console.log(self.resourceUrl);
+      window.open('file/download?url=' + self.resourceUrl);
+    };
+  });
+</script>
+</body>
+</html>

+ 31 - 0
oos-dashboard/src/main/resources/templates/index.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8"/>
+  <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
+  <title>Home(Dashboard)</title>
+  <meta name="description" content=""/>
+  <meta name="viewport" content="width=device-width"/>
+  <base href="/"/>
+  <link rel="stylesheet" type="text/css"
+        href="/webjars/bootstrap/css/bootstrap.min.css"/>
+</head>
+<body>
+<div class="container">
+  <h1>Home</h1>
+  <div class="row">
+    <!-- interface list -->
+    <div class="col-md-6 col-sm-12">
+      <h3>接口测试</h3>
+      <ol>
+        <li><a href="download">文件下载</a></li>
+        <li><a href="upload">文件上传</a></li>
+      </ol>
+    </div>
+  </div>
+</div>
+
+<script type="text/javascript" src="/webjars/jquery/jquery.min.js"></script>
+<script type="text/javascript" src="/webjars/bootstrap/js/bootstrap.min.js"></script>
+</body>
+</html>

+ 142 - 0
oos-dashboard/src/main/resources/templates/upload.html

@@ -0,0 +1,142 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8"/>
+  <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
+  <title>Upload(Dashboard)</title>
+  <meta name="description" content=""/>
+  <meta name="viewport" content="width=device-width"/>
+  <base href="/"/>
+  <link rel="stylesheet" type="text/css"
+        href="/webjars/bootstrap/css/bootstrap.min.css"/>
+</head>
+<body>
+<div class="container" ng-app="app">
+  <h1>文件上传
+    <small><a href="index">首页</a></small>
+  </h1>
+  <!-- file upload -->
+  <div style="margin-top: 40px;margin-bottom: 40px;" ng-controller="UploadController as upload">
+    <div class="row">
+      <!-- file upload area -->
+      <div class="col-sm-6">
+        <form>
+          <div class="form-group" style="margin-top: 20px; margin-bottom: 20px;">
+            <label for="localFile">本地文件上传</label>
+            <input type="file" id="localFile" name="file"/>
+          </div>
+          <button class="btn btn-default" ng-click="upload.uploadLocalFile()">本地上传</button>
+
+          <div class="form-group" style="margin-top: 20px; margin-bottom: 20px;">
+            <label for="remoteFile">远程文件上传</label>
+            <input type="url" class="form-control" id="remoteFile" name="url" placeholder="URL"
+              ng-model="upload.remoteUrl"/>
+          </div>
+          <button class="btn btn-default" ng-click="upload.uploadRemoteFile()">远程上传</button>
+        </form>
+      </div>
+      <!-- file show area -->
+      <div class="col-sm-6">
+        <div ng-if="upload.errorMessage">
+          <label ng-bind="upload.errorMessage" style="color: red"></label>
+        </div>
+        <div ng-if="!upload.errorMessage">
+          <div><label ng-bind="upload.url"></label></div>
+          <img class="img-rounded" ng-src="{{upload.url}}" alt="上传图片" width="200" height="200" />
+        </div>
+      </div>
+    </div>
+  </div>
+  <a href="download">文件下载</a>
+</div>
+
+<!-- Scripts -->
+<script type="text/javascript" src="/webjars/jquery/jquery.min.js"></script>
+<script type="text/javascript" src="/webjars/bootstrap/js/bootstrap.min.js"></script>
+<script type="text/javascript" src="/webjars/angularjs/angular.min.js"></script>
+<script type="text/javascript">
+  angular.module("app", [])
+    .config(function ($httpProvider) {
+      $httpProvider.defaults.headers.common['X-Requested-With'] = 'XMLHttpRequest';
+    }).controller("UploadController", function ($http, $location) {
+      const self = this;
+
+      self.url = 'N/A';
+      self.errorMessage = undefined;
+
+      self.remoteUrl = '';
+
+      self.uploadLocalFile = function () {
+        const fd = new FormData();
+        const file = document.querySelector('input[type=file]').files[0];
+        fd.append("file", file);
+
+        $http({
+          method: 'POST',
+          url: 'file/upload',
+          data: fd,
+          params: { bucketName: 'uas.test' },
+          headers: { 'Content-Type': undefined },
+          transformRequest: angular.identity
+        }).success(function (response) {
+          console.log(response);
+          if (response.success) {
+            self.errorMessage = null;
+            self.url = response.data;
+
+            document.querySelector('input[type=file]').files[0] = null;
+          } else {
+            self.errorMessage = response.message;
+          }
+        }).error(function (response) {
+          console.log(response);
+          self.url = 'N/A';
+          if (response.status) {
+            self.errorMessage = response.status + ":" + response.message;
+            alert(response.message);
+          } else {
+            self.errorMessage = response.message;
+            alert(response.message);
+          }
+        })
+      };
+
+      self.uploadRemoteFile = function () {
+        const params = { bucketName: 'b2c.test', url: self.remoteUrl };
+
+        $http({
+          method: 'POST',
+          url: 'file/upload-remote',
+          params: params,
+          transformRequest: angular.identity
+        }).success(function (response) {
+          console.log(response);
+          if (response.success) {
+            self.errorMessage = null;
+            self.url = response.data;
+
+            self.remoteUrl = '';
+          } else {
+            self.url = 'N/A';
+            self.errorMessage = response.message;
+
+            self.remoteUrl = '';
+          }
+        }).error(function (response) {
+          console.log(response);
+          self.url = 'N/A';
+          self.remoteUrl = '';
+          if (response.status) {
+            self.errorMessage = response.status + ":" + response.message;
+            alert(response.message);
+          } else {
+            self.errorMessage = response.message;
+            alert(response.message);
+          }
+        })
+      };
+    });
+</script>
+
+</body>
+</html>

+ 48 - 0
oos-migration/build.gradle

@@ -0,0 +1,48 @@
+buildscript {
+    ext {
+        springBootVersion = '1.5.7.RELEASE'
+        dockerVersion = '0.12.0'
+        dcokerRegistry = "10.10.100.200:5000"
+    }
+    repositories {
+        mavenLocal()
+        maven { url "https://plugins.gradle.org/m2/" }
+        maven { url "http://maven.aliyun.com/nexus/content/groups/public/" }
+        maven { url 'http://10.10.101.21:8081/artifactory/ext-release-local' }
+        jcenter()
+        mavenCentral()
+    }
+    dependencies {
+        classpath "org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}"
+        classpath "gradle.plugin.com.palantir.gradle.docker:gradle-docker:${dockerVersion}"
+    }
+}
+
+group 'com.uas.platform.oos'
+version '1.0.1-SNAPSHOT'
+
+apply plugin: "com.palantir.docker"
+apply plugin: 'org.springframework.boot'
+
+apply from: "$rootDir/gradle/tasks.gradle"
+
+dependencies {
+    compile project(":oos-core")
+    compile ("com.google.guava:guava:23.0")
+    compile "org.springframework.boot:spring-boot-starter-web"
+    compile "org.springframework.boot:spring-boot-starter-thymeleaf"
+    compile "org.springframework.boot:spring-boot-starter-data-jpa"
+    compile "mysql:mysql-connector-java:6.0.6"
+
+    compile "org.webjars:webjars-locator"
+    compile "org.webjars:jquery:2.1.1"
+    compile "org.webjars:angularjs:1.4.3"
+    compile "org.webjars:bootstrap:3.2.0"
+
+    testCompile "org.springframework.boot:spring-boot-starter-test"
+}
+
+jar {
+    baseName = project.name
+    version = ''
+}

+ 6 - 0
oos-migration/src/main/docker/Dockerfile

@@ -0,0 +1,6 @@
+FROM hub.c.163.com/library/java:8-jre-alpine
+VOLUME /tmp
+ADD oos-dashboard.jar app.jar
+RUN sh -c "touch /app.jar"
+ENV JAVA_OPTS=""
+ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar /app.jar"]

+ 26 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/MigrationApplication.java

@@ -0,0 +1,26 @@
+package com.uas.platform.oos.migration;
+
+import org.springframework.boot.Banner.Mode;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+
+/**
+ * Bootstrap dashboard application.
+ *
+ * @author huxz
+ */
+@SpringBootApplication
+public class MigrationApplication {
+
+  /**
+   * Application entry.
+   *
+   * @param args    command line parameters
+   */
+  public static void main(String[] args) {
+    new SpringApplicationBuilder(MigrationApplication.class)
+        .bannerMode(Mode.OFF)
+        .web(true)
+        .run(args);
+  }
+}

+ 62 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/api/MigrationController.java

@@ -0,0 +1,62 @@
+package com.uas.platform.oos.migration.api;
+
+import com.uas.platform.oos.core.common.ResultBean;
+import com.uas.platform.oos.migration.entity.OperationResult;
+import com.uas.platform.oos.migration.service.MigrationFileService;
+import com.uas.platform.oos.migration.service.impl.MigrationFileServiceImpl.MigrationStatistic;
+import java.util.List;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.web.PageableDefault;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping(value = "/api/file/migration")
+public class MigrationController {
+
+  private final MigrationFileService migrationFileService;
+
+  @Autowired
+  public MigrationController(
+      MigrationFileService migrationFileService) {
+    this.migrationFileService = migrationFileService;
+  }
+
+  @PostMapping(value = "//create-records", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
+  public ResultBean<Long> createMigrationRecords(String tableName, String fieldName) {
+    return new ResultBean<>(migrationFileService.createMigrationFileLog(tableName, fieldName));
+  }
+
+  @GetMapping(value = "//migrate-statistic", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
+  public ResultBean<OperationResult> gainStatisticsForMigration(String tableName, String fieldName) {
+    return new ResultBean<>(migrationFileService.gainStatisticsForMigration(tableName, fieldName));
+  }
+
+  @PostMapping(value = "//move-file", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
+  public ResultBean<Long> moveFileToOos(@PageableDefault(size = 200) Pageable pageable) {
+    return new ResultBean<>(migrationFileService.migrateFileToOos(pageable));
+  }
+
+  /**
+   * gain statistic After migrated.
+   */
+  @GetMapping(value = "//migration-statistic", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
+  public ResultBean<List<MigrationStatistic>> gainStatisticsForMovingFile() {
+    return new ResultBean<>(migrationFileService.gainStatisticsForMovingFile());
+  }
+
+  @GetMapping(value = "//gain-fields", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
+  public ResultBean<List<String>> gainFieldsWillMigrate() {
+    return new ResultBean<>(migrationFileService.gainFieldsWillMigrate());
+  }
+
+  @PutMapping(value = "//sync-links", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
+  public ResultBean<Long> syncResourceLink(String tableName, String fieldName) {
+    return new ResultBean<>(migrationFileService.syncResourceLink(tableName, fieldName));
+  }
+}

+ 43 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/config/ControllerExceptionHandler.java

@@ -0,0 +1,43 @@
+package com.uas.platform.oos.migration.config;
+
+import com.uas.platform.oos.core.common.ResultBean;
+import java.io.IOException;
+import org.apache.log4j.Logger;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.http.converter.HttpMessageConverter;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
+
+/**
+ * Use @ControllerAdvice annotation to handler exceptions.
+ *
+ * <p>Tip: ResponseEntityExceptionHandler, write to the response with a {@link HttpMessageConverter
+ * message converter}</p>
+ *
+ * @author huxz
+ */
+@ControllerAdvice(annotations = {RestController.class})
+public class ControllerExceptionHandler extends ResponseEntityExceptionHandler {
+
+  private static final Logger logger = Logger.getLogger(ControllerExceptionHandler.class);
+
+  /**
+   * Handler IOException and RuntimeException.
+   *
+   * @param ex exception info
+   */
+  @ExceptionHandler(value = {IOException.class, RuntimeException.class})
+  @ResponseBody
+  public ResponseEntity<?> handlerCommonExceptions(Throwable ex) {
+    logger.error("System Error occurs", ex);
+    HttpHeaders headers = new HttpHeaders();
+    headers.add("Content-Type", MediaType.APPLICATION_JSON_UTF8_VALUE);
+    return new ResponseEntity<>(new ResultBean<>(ex), headers, HttpStatus.INTERNAL_SERVER_ERROR);
+  }
+}

+ 23 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/dao/MigrationFileRepository.java

@@ -0,0 +1,23 @@
+package com.uas.platform.oos.migration.dao;
+
+import com.uas.platform.oos.migration.entity.MigrationFile;
+import com.uas.platform.oos.migration.entity.MigrationFile.FileStatus;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface MigrationFileRepository extends JpaSpecificationExecutor<MigrationFile>,
+    JpaRepository<MigrationFile, Long> {
+
+  MigrationFile findByOldUrlOrNewUrl(String oldUrl, String newUrl);
+
+  MigrationFile findByOldUrl(String oldUrl);
+
+  Long countByStatus(FileStatus status);
+
+  Page<MigrationFile> readByStatus(FileStatus status, Pageable pageable);
+
+}

+ 12 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/dao/MockUrlRepository.java

@@ -0,0 +1,12 @@
+package com.uas.platform.oos.migration.dao;
+
+import com.uas.platform.oos.migration.entity.MockUrl;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface MockUrlRepository extends JpaSpecificationExecutor<MockUrl>,
+    JpaRepository<MockUrl, Long> {
+
+}

+ 29 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/dao/ResourceFieldsRepository.java

@@ -0,0 +1,29 @@
+package com.uas.platform.oos.migration.dao;
+
+import com.uas.platform.oos.migration.entity.MigrationFile;
+import com.uas.platform.oos.migration.entity.MigrationFile.FileStatus;
+import com.uas.platform.oos.migration.entity.ResourceFields;
+import java.util.List;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface ResourceFieldsRepository extends JpaRepository<ResourceFields, Long> {
+
+  ResourceFields findByFieldNameAndMigration(String fieldName, MigrationFile migration);
+
+  Long countByFieldName(String fieldName);
+
+  @Query(value = "select count(*) from ResourceFields r where r.fieldName = :fieldName and r.status = :status")
+  Long countByFieldNameAndStatus(@Param("fieldName") String fieldName,
+      @Param("status") FileStatus status);
+
+  List<ResourceFields> findByMigration(MigrationFile migration);
+
+  Page<ResourceFields> readByFieldNameAndStatus(String fieldName, FileStatus status,
+      Pageable pageable);
+}

+ 118 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/entity/MigrationFile.java

@@ -0,0 +1,118 @@
+package com.uas.platform.oos.migration.entity;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.uas.platform.oos.core.util.JacksonUtils;
+import java.util.Date;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+/**
+ * store to be migrated files.
+ *
+ * @author huxz
+ */
+@Entity
+@Table(name = "a_migration")
+@JsonInclude(Include.NON_EMPTY)
+public class MigrationFile {
+
+  @Id
+  @GeneratedValue(strategy = GenerationType.AUTO)
+  private Long id;
+
+  @Column(name = "old_url", unique = true)
+  private String oldUrl;
+
+  @Column(name = "new_url", unique = true)
+  private String newUrl;
+
+  @Column(name = "create_time")
+  private Date createTime;
+
+  @Column(name = "migrate_time")
+  private Date migrateTime;
+
+  @Column(name = "migration_status")
+  @Enumerated(EnumType.STRING)
+  private FileStatus status;
+
+  @Column(name = "reason", length = 3000)
+  private String reason;
+
+  public enum FileStatus {
+    CREATED,          // 创建文件迁移记录
+    MIGRATED,         // 文件迁移成功
+    SYNCHRONIZED      // 同步文件链接成功
+  }
+
+  public MigrationFile() {
+  }
+
+  public Long getId() {
+    return id;
+  }
+
+  public void setId(Long id) {
+    this.id = id;
+  }
+
+  public String getOldUrl() {
+    return oldUrl;
+  }
+
+  public void setOldUrl(String oldUrl) {
+    this.oldUrl = oldUrl;
+  }
+
+  public String getNewUrl() {
+    return newUrl;
+  }
+
+  public void setNewUrl(String newUrl) {
+    this.newUrl = newUrl;
+  }
+
+  public Date getCreateTime() {
+    return createTime;
+  }
+
+  public void setCreateTime(Date createTime) {
+    this.createTime = createTime;
+  }
+
+  public Date getMigrateTime() {
+    return migrateTime;
+  }
+
+  public void setMigrateTime(Date migrateTime) {
+    this.migrateTime = migrateTime;
+  }
+
+  public FileStatus getStatus() {
+    return status;
+  }
+
+  public void setStatus(FileStatus status) {
+    this.status = status;
+  }
+
+  public String getReason() {
+    return reason;
+  }
+
+  public void setReason(String reason) {
+    this.reason = reason;
+  }
+
+  @Override
+  public String toString() {
+    return JacksonUtils.toJson(this);
+  }
+}

+ 67 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/entity/MockUrl.java

@@ -0,0 +1,67 @@
+package com.uas.platform.oos.migration.entity;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.uas.platform.oos.core.util.JacksonUtils;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+/**
+ * Mock url.
+ *
+ * @author huxz
+ */
+@Entity
+@Table(name = "a_mock_url")
+@JsonInclude(Include.NON_EMPTY)
+public class MockUrl {
+
+  @Id
+  @GeneratedValue(strategy = GenerationType.AUTO)
+  private Long id;
+
+  /**
+   * file link.
+   */
+  private String url;
+
+  /**
+   * picture link.
+   */
+  private String pictureUrl;
+
+  public MockUrl() {
+  }
+
+  public Long getId() {
+    return id;
+  }
+
+  public void setId(Long id) {
+    this.id = id;
+  }
+
+  public String getUrl() {
+    return url;
+  }
+
+  public void setUrl(String url) {
+    this.url = url;
+  }
+
+  public String getPictureUrl() {
+    return pictureUrl;
+  }
+
+  public void setPictureUrl(String pictureUrl) {
+    this.pictureUrl = pictureUrl;
+  }
+
+  @Override
+  public String toString() {
+    return JacksonUtils.toJson(this);
+  }
+}

+ 37 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/entity/OperationResult.java

@@ -0,0 +1,37 @@
+package com.uas.platform.oos.migration.entity;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.uas.platform.oos.core.util.JacksonUtils;
+
+@JsonInclude(Include.NON_EMPTY)
+public class OperationResult {
+
+  private Long total;
+
+  private Long todo;
+
+  public OperationResult() {
+  }
+
+  public Long getTotal() {
+    return total;
+  }
+
+  public void setTotal(Long total) {
+    this.total = total;
+  }
+
+  public Long getTodo() {
+    return todo;
+  }
+
+  public void setTodo(Long todo) {
+    this.todo = todo;
+  }
+
+  @Override
+  public String toString() {
+    return JacksonUtils.toJson(this);
+  }
+}

+ 99 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/entity/ResourceFields.java

@@ -0,0 +1,99 @@
+package com.uas.platform.oos.migration.entity;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.uas.platform.oos.core.util.JacksonUtils;
+import com.uas.platform.oos.migration.entity.MigrationFile.FileStatus;
+import java.util.Date;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+import javax.persistence.Index;
+import javax.persistence.JoinColumn;
+import javax.persistence.OneToOne;
+import javax.persistence.Table;
+
+/**
+ * Record one resource is in multi field of different table.
+ *
+ * @author huxz
+ */
+@Entity
+@Table(name = "a_resource_to_fields", indexes = {
+    @Index(name = "field_resource", columnList = "field_name, migration_id", unique = true)
+})
+@JsonInclude(Include.NON_EMPTY)
+public class ResourceFields {
+
+  @Id
+  @GeneratedValue
+  private Long id;
+
+  /**
+   * Example: tableName.fieldName
+   */
+  @Column(name = "field_name")
+  private String fieldName;
+
+  @OneToOne
+  @JoinColumn(name = "migration_id")
+  private MigrationFile migration;
+
+  @Column(name = "create_time")
+  private Date createTime;
+
+  @Column(name = "mapping_status")
+  @Enumerated(EnumType.STRING)
+  private FileStatus status;
+
+  public ResourceFields() {
+  }
+
+  public Long getId() {
+    return id;
+  }
+
+  public void setId(Long id) {
+    this.id = id;
+  }
+
+  public String getFieldName() {
+    return fieldName;
+  }
+
+  public void setFieldName(String fieldName) {
+    this.fieldName = fieldName;
+  }
+
+  public MigrationFile getMigration() {
+    return migration;
+  }
+
+  public void setMigration(MigrationFile migration) {
+    this.migration = migration;
+  }
+
+  public Date getCreateTime() {
+    return createTime;
+  }
+
+  public void setCreateTime(Date createTime) {
+    this.createTime = createTime;
+  }
+
+  public FileStatus getStatus() {
+    return status;
+  }
+
+  public void setStatus(FileStatus status) {
+    this.status = status;
+  }
+
+  @Override
+  public String toString() {
+    return JacksonUtils.toJson(this);
+  }
+}

+ 51 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/service/MigrationFileService.java

@@ -0,0 +1,51 @@
+package com.uas.platform.oos.migration.service;
+
+import com.uas.platform.oos.migration.entity.MigrationFile;
+import com.uas.platform.oos.migration.entity.OperationResult;
+import com.uas.platform.oos.migration.service.impl.MigrationFileServiceImpl.MigrationStatistic;
+import java.util.List;
+import org.springframework.data.domain.Pageable;
+
+public interface MigrationFileService {
+
+  /**
+   * Create a migration file log.
+   *
+   * @param tableName   the name of table.
+   * @param fieldName   the name of field.
+   */
+  Long createMigrationFileLog(String tableName, String fieldName);
+
+  /**
+   * Gain statistics for migration.
+   *
+   * @param tableName   the name of table
+   * @param fieldName   the name of field
+   */
+  OperationResult gainStatisticsForMigration(String tableName, String fieldName);
+
+  /**
+   * Save migration file log to database.
+   *
+   * @param url     the url of resource
+   */
+  MigrationFile saveMigrationFile(String url);
+
+  /**
+   * migrate files to OOS.
+   * @param pageable
+   */
+  Long migrateFileToOos(Pageable pageable);
+
+  /**
+   * Gain statistics for moving files.
+   */
+  List<MigrationStatistic> gainStatisticsForMovingFile();
+
+  /**
+   * Gain fields will be migrating.
+   */
+  List<String> gainFieldsWillMigrate();
+
+  Long syncResourceLink(String tableName, String fieldName);
+}

+ 10 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/service/MockUrlService.java

@@ -0,0 +1,10 @@
+package com.uas.platform.oos.migration.service;
+
+public interface MockUrlService {
+
+  /**
+   * Mock urls to test file service.
+   */
+  void mockBatchUrl();
+
+}

+ 452 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/service/impl/MigrationFileServiceImpl.java

@@ -0,0 +1,452 @@
+package com.uas.platform.oos.migration.service.impl;
+
+import static com.uas.platform.oos.core.validate.ArgumentValidate.assertParameterNotNull;
+import static com.uas.platform.oos.core.validate.ArgumentValidate.assertStringNotEmpty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.uas.platform.oos.core.oos.client.OosClientHelper;
+import com.uas.platform.oos.core.util.JacksonUtils;
+import com.uas.platform.oos.migration.dao.MigrationFileRepository;
+import com.uas.platform.oos.migration.dao.ResourceFieldsRepository;
+import com.uas.platform.oos.migration.entity.MigrationFile;
+import com.uas.platform.oos.migration.entity.MigrationFile.FileStatus;
+import com.uas.platform.oos.migration.entity.OperationResult;
+import com.uas.platform.oos.migration.entity.ResourceFields;
+import com.uas.platform.oos.migration.service.MigrationFileService;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor.AbortPolicy;
+import java.util.concurrent.TimeUnit;
+import org.apache.log4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Pageable;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
+
+@Service
+public class MigrationFileServiceImpl implements MigrationFileService {
+
+  private static final String BASE_URL = "https://oos-gz.ctyunapi.cn";
+
+  private static final String QUERY_RESOURCE_URL_STATEMENT = "SELECT COUNT(DISTINCT %s) "
+      + "FROM %s WHERE %s NOT LIKE ?";
+
+  private static final String BUCKET_NAME = "b2c.test";
+
+  private final Logger logger = Logger.getLogger(getClass());
+
+  private final MigrationFileRepository migrationRepository;
+
+  private final ResourceFieldsRepository fieldsRepository;
+
+  private final JdbcTemplate jdbcTemplate;
+
+  /**
+   * Constructor.
+   */
+  @Autowired
+  public MigrationFileServiceImpl(
+      MigrationFileRepository migrationRepository,
+      ResourceFieldsRepository fieldsRepository,
+      JdbcTemplate jdbcTemplate) {
+    this.migrationRepository = migrationRepository;
+    this.fieldsRepository = fieldsRepository;
+    this.jdbcTemplate = jdbcTemplate;
+  }
+
+  @Override
+  public Long createMigrationFileLog(final String tableName, final String fieldName) {
+    assertStringNotEmpty(tableName, "The name of table to be migrating must be non-empty");
+    assertStringNotEmpty(tableName, "The name of table field to be migrating must be non-empty");
+
+    final LocalDateTime startTime = LocalDateTime.now();
+
+    // Count total number of records to be handled
+    Object[] parameters = new Object[]{BASE_URL + "%"};
+    String countSql = String.format(QUERY_RESOURCE_URL_STATEMENT, fieldName, tableName, fieldName);
+    Long count = jdbcTemplate.queryForObject(countSql, parameters, Long.class);
+
+    if (logger.isDebugEnabled()) {
+      logger.info(String.format("%s.%s has %d records to be handled", tableName, fieldName, count));
+    }
+
+    // return if the total number of records to be handled is 0
+    if (count == null || count <= 0) {
+      return (long) 0;
+    }
+
+    // calculate page information of data set.
+    final int pageSize = 500;
+    int totalPage = calculateTotalPage(count, pageSize);
+
+    CountDownLatch latch = new CountDownLatch(totalPage);
+
+    ExecutorService executorService = getExecutorService();
+
+    for (int i = 0; i < totalPage; i++) {
+      final int minIndex = i * pageSize;
+      executorService.execute(() -> {
+        batchCreateMigrateRecords(tableName, fieldName, minIndex, pageSize, parameters);
+        latch.countDown();
+      });
+    }
+
+    try {
+      latch.await(60, TimeUnit.SECONDS);
+      executorService.shutdown();
+    } catch (InterruptedException e) {
+      logger.error("Concurrent tasks occurs error", e);
+    }
+
+    LocalDateTime endTime = LocalDateTime.now();
+    Duration between = Duration.between(startTime, endTime);
+    return between.toMillis();
+  }
+
+  private ExecutorService getExecutorService() {
+    ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("migration-pool-%d")
+        .build();
+    return new ThreadPoolExecutor(5, 200, 0L, TimeUnit.MILLISECONDS,
+        new LinkedBlockingDeque<>(), threadFactory, new AbortPolicy());
+  }
+
+  /**
+   * Calculate total page according to total number and page size.
+   *
+   * @param count the total number of all records
+   * @param pageSize the size of page
+   */
+  private int calculateTotalPage(Long count, int pageSize) {
+    int totalPage = (int) ((count - 1) / pageSize) + 1;
+
+    if (logger.isDebugEnabled()) {
+      logger.debug(String.format("Total: %d, PageSize: %d, TotalPage: %d",
+          count, pageSize, totalPage));
+    }
+    return totalPage;
+  }
+
+  /**
+   * Create a batch of migrate records.
+   *
+   * @param tableName the name of table migrated
+   * @param fieldName the name of field migrated
+   * @param minIndex the first index of current page
+   * @param pageSize the size of per page
+   * @param parameters sql parameters
+   */
+  @Transactional(rollbackFor = RuntimeException.class)
+  public void batchCreateMigrateRecords(String tableName, String fieldName, int minIndex,
+      int pageSize, Object... parameters) {
+    String sql = String.format("SELECT DISTINCT %s FROM %s WHERE %s NOT LIKE ? limit %d, %d",
+        fieldName, tableName, fieldName, minIndex, pageSize);
+    if (logger.isDebugEnabled()) {
+      logger.debug("Dynamic SQL: " + sql);
+    }
+
+    List<FileUrl> fileUrls = jdbcTemplate.query(sql, parameters, (rs, rowNum) -> {
+      FileUrl fileUrl = new FileUrl();
+      fileUrl.setUrl(rs.getString(fieldName));
+      return fileUrl;
+    });
+    logger.info("The number of current page records: " + fileUrls.size());
+
+    for (FileUrl fileUrl : fileUrls) {
+      MigrationFile file = saveMigrationFile(fileUrl.getUrl());
+
+      String fieldFullName = tableName.toLowerCase() + "." + fieldName.toUpperCase();
+      ResourceFields exist = fieldsRepository.findByFieldNameAndMigration(fieldFullName, file);
+      if (exist == null) {
+        ResourceFields fields = new ResourceFields();
+        fields.setFieldName(fieldFullName);
+        fields.setMigration(file);
+        fields.setCreateTime(new Date());
+        fields.setStatus(file.getStatus());
+        fieldsRepository.save(fields);
+      } else {
+        if (logger.isDebugEnabled()) {
+          logger.info("One resource related to this field exists");
+        }
+      }
+    }
+  }
+
+  @Override
+  public MigrationFile saveMigrationFile(String url) {
+    final Date date = new Date();
+    MigrationFile migrationFile = migrationRepository.findByOldUrl(url);
+    if (migrationFile != null) {
+      return migrationFile;
+    }
+
+    MigrationFile file = new MigrationFile();
+    file.setOldUrl(url);
+    file.setStatus(FileStatus.CREATED);
+    file.setCreateTime(new Date());
+    migrationRepository.save(file);
+    return file;
+  }
+
+  @Override
+  public OperationResult gainStatisticsForMigration(String tableName, String fieldName) {
+    assertStringNotEmpty(tableName, "The name of table to be migrating must be non-empty");
+    assertStringNotEmpty(tableName, "The name of table field to be migrating must be non-empty");
+
+    // Count total number of records to be handled
+    Object[] parameters = new Object[]{BASE_URL + "%"};
+    String countSql = String.format("SELECT COUNT(DISTINCT %s) FROM %s WHERE %s NOT LIKE ?",
+        fieldName, tableName, fieldName);
+    Long count = jdbcTemplate.queryForObject(countSql, parameters, Long.class);
+
+    String fieldFullName = tableName.toLowerCase() + "." + fieldName.toUpperCase();
+    Long resourceCount = fieldsRepository.countByFieldName(fieldFullName);
+
+    OperationResult result = new OperationResult();
+    result.setTotal(count == null || count <= 0 ? 0 : count);
+    result.setTodo(resourceCount);
+    return result;
+  }
+
+  @Override
+  public Long migrateFileToOos(Pageable pageable) {
+    assertParameterNotNull(pageable, "Page parameter must be non-empty");
+    final LocalDateTime startTime = LocalDateTime.now();
+
+    Page<MigrationFile> filePage = migrationRepository.readByStatus(FileStatus.CREATED, pageable);
+
+    if (filePage == null || CollectionUtils.isEmpty(filePage.getContent())) {
+      return (long) 0;
+    }
+
+    // Use concurrent package to reduce time migrate files
+    CountDownLatch latch = new CountDownLatch(filePage.getContent().size());
+    ExecutorService executor = getExecutorService();
+    for (MigrationFile file : filePage.getContent()) {
+      executor.execute(() -> {
+        try {
+          migrateFileToOos(file);
+        } finally {
+          latch.countDown();
+        }
+      });
+
+    }
+    executor.shutdown();
+
+    try {
+      latch.await(3, TimeUnit.MINUTES);
+      logger.info("The Count of file: " + filePage.getContent().size());
+    } catch (InterruptedException e) {
+      logger.error("Task interrupt", e);
+    }
+
+    LocalDateTime endTime = LocalDateTime.now();
+    return Duration.between(startTime, endTime).toMillis();
+  }
+
+  @Transactional(rollbackFor = RuntimeException.class)
+  public void migrateFileToOos(MigrationFile file) {
+    if (logger.isDebugEnabled()) {
+      logger.debug(String.format("Upload file[%s]", file.getOldUrl()));
+    }
+
+    // upload file
+    OosClientHelper helper = new OosClientHelper();
+    String url = helper.upload(BUCKET_NAME, file.getOldUrl());
+    if (logger.isDebugEnabled()) {
+      if (!StringUtils.isEmpty(url)) {
+        logger.debug(String.format("File[%s] upload successfully, new url[%s]", file.getOldUrl(),
+            url));
+      }
+    }
+
+    file.setNewUrl(url);
+    file.setMigrateTime(new Date());
+    file.setStatus(FileStatus.MIGRATED);
+    migrationRepository.save(file);
+
+    // Set Mapping Status
+    List<ResourceFields> fieldsList = fieldsRepository.findByMigration(file);
+    if (!CollectionUtils.isEmpty(fieldsList)) {
+      for (ResourceFields fields : fieldsList) {
+        fields.setStatus(FileStatus.MIGRATED);
+      }
+      fieldsRepository.save(fieldsList);
+    }
+  }
+
+  @Override
+  public List<MigrationStatistic> gainStatisticsForMovingFile() {
+    String mappingTableName = "a_resource_to_fields";
+    String sql = "SELECT field_name, mapping_status, count(*) as total_number FROM "
+        + mappingTableName + " GROUP BY field_name, mapping_status";
+
+    return jdbcTemplate.query(sql,
+        (rs, rowNum) -> {
+          MigrationStatistic statistic = new MigrationStatistic();
+          statistic.setFieldName(rs.getString("field_name"));
+          statistic.setStatus(rs.getString("mapping_status"));
+          statistic.setTotal(rs.getLong("total_number"));
+          return statistic;
+        });
+  }
+
+  @Override
+  public List<String> gainFieldsWillMigrate() {
+    String mappingTableName = "a_resource_to_fields";
+    String sql = "SELECT DISTINCT field_name FROM " + mappingTableName
+        + " WHERE mapping_status = 'MIGRATED'";
+
+    return jdbcTemplate.query(sql, (rs, rowNum) -> rs.getString("field_name"));
+  }
+
+  @Override
+  public Long syncResourceLink(String tableName, String fieldName) {
+    assertStringNotEmpty(tableName, "The name of table to be migrating must be non-empty");
+    assertStringNotEmpty(tableName, "The name of table field to be migrating must be non-empty");
+
+    if (logger.isDebugEnabled()) {
+      logger.debug(String
+          .format("Sync links of field: %s.%s", tableName.toLowerCase(), fieldName.toUpperCase()));
+    }
+
+    final LocalDateTime startTime = LocalDateTime.now();
+
+    // Count total number of records to be handled
+    final String fieldFullName = tableName + "." + fieldName;
+    Long count = fieldsRepository.countByFieldNameAndStatus(fieldFullName, FileStatus.MIGRATED);
+    if (logger.isDebugEnabled()) {
+      logger.debug(String.format("%s.%s has %d records migrated", tableName, fieldName, count));
+    }
+
+    // return if the total number of records to be handled is 0
+    if (count == null || count <= 0) {
+      return 0L;
+    }
+
+    // calculate page information of data set.
+    final int pageSize = 500;
+    int totalPage = calculateTotalPage(count, pageSize);
+    if (logger.isDebugEnabled()) {
+      logger.debug("Print total page: " + totalPage);
+    }
+
+    CountDownLatch latch = new CountDownLatch(totalPage);
+    ExecutorService executorService = getExecutorService();
+    for (int i = 0; i < totalPage; i++) {
+      final int index = i;
+      executorService.execute(() -> {
+        try {
+          Pageable pageInfo = new PageRequest(index, pageSize);
+
+          Page<ResourceFields> fieldsPage = fieldsRepository
+              .readByFieldNameAndStatus(fieldFullName, FileStatus.MIGRATED, pageInfo);
+
+          if (!CollectionUtils.isEmpty(fieldsPage.getContent())) {
+            for (ResourceFields fields : fieldsPage.getContent()) {
+              syncResourceLinkOfField(fields, tableName, fieldName);
+            }
+          }
+        } finally {
+          latch.countDown();
+        }
+      });
+    }
+    executorService.shutdown();
+
+    try {
+      latch.await(3, TimeUnit.MINUTES);
+      logger.info("Task finished");
+    } catch (InterruptedException e) {
+      logger.error("Concurrent tasks occurs error", e);
+    }
+
+    LocalDateTime endTime = LocalDateTime.now();
+    return Duration.between(startTime, endTime).toMillis();
+  }
+
+  @Transactional(rollbackFor = RuntimeException.class)
+  public void syncResourceLinkOfField(ResourceFields fields, String tableName, String fieldName) {
+    String oldUrl = fields.getMigration().getOldUrl();
+    String newUrl = fields.getMigration().getNewUrl();
+
+    Object[] parameters = new Object[]{newUrl, oldUrl};
+    String sql = String
+        .format("UPDATE %s SET %s = ? WHERE %s = ?", tableName, fieldName, fieldName);
+    int result = jdbcTemplate.update(sql, parameters);
+    logger.info("Update " + result + " rows");
+
+    fields.setStatus(FileStatus.SYNCHRONIZED);
+    fieldsRepository.save(fields);
+  }
+
+  @JsonInclude(Include.NON_EMPTY)
+  public class MigrationStatistic {
+
+    private String fieldName;
+
+    private String status;
+
+    private Long total;
+
+    public MigrationStatistic() {
+    }
+
+    public String getFieldName() {
+      return fieldName;
+    }
+
+    public void setFieldName(String fieldName) {
+      this.fieldName = fieldName;
+    }
+
+    public String getStatus() {
+      return status;
+    }
+
+    public void setStatus(String status) {
+      this.status = status;
+    }
+
+    public Long getTotal() {
+      return total;
+    }
+
+    public void setTotal(Long total) {
+      this.total = total;
+    }
+
+    @Override
+    public String toString() {
+      return JacksonUtils.toJson(this);
+    }
+  }
+
+  private class FileUrl {
+
+    private String url;
+
+    String getUrl() {
+      return url;
+    }
+
+    void setUrl(String url) {
+      this.url = url;
+    }
+  }
+
+}

+ 43 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/service/impl/MockUrlServiceImpl.java

@@ -0,0 +1,43 @@
+package com.uas.platform.oos.migration.service.impl;
+
+import com.uas.platform.oos.migration.dao.MockUrlRepository;
+import com.uas.platform.oos.migration.entity.MockUrl;
+import com.uas.platform.oos.migration.service.MockUrlService;
+import java.util.UUID;
+import javax.transaction.Transactional;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+@Service
+public class MockUrlServiceImpl implements MockUrlService {
+
+  private final MockUrlRepository mockUrlRepository;
+
+  @Autowired
+  public MockUrlServiceImpl(MockUrlRepository mockUrlRepository) {
+    this.mockUrlRepository = mockUrlRepository;
+  }
+
+  @Override
+  public void mockBatchUrl() {
+    String baseUrl = "http://www.us.com";
+
+    for (int i = 0; i < 10000; i++) {
+      String uuid = UUID.randomUUID().toString().replace("-", "");
+      saveMockUrl(baseUrl + "/image/" + uuid + ".jpg");
+    }
+  }
+
+  /**
+   * Save mocked url to database.
+   *
+   * @param url     mocked url
+   */
+  @Transactional
+  public void saveMockUrl(String url) {
+    MockUrl mockUrl = new MockUrl();
+    mockUrl.setUrl(url);
+    mockUrl.setPictureUrl(url);
+    mockUrlRepository.save(mockUrl);
+  }
+}

+ 31 - 0
oos-migration/src/main/java/com/uas/platform/oos/migration/web/HomeController.java

@@ -0,0 +1,31 @@
+package com.uas.platform.oos.migration.web;
+
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+
+@Controller
+@RequestMapping(value = "/")
+public class HomeController {
+
+  @RequestMapping(method = RequestMethod.GET, value = {"index", ""})
+  public String home() {
+    return "index";
+  }
+
+  @GetMapping(value = "migration/create-record")
+  public String addToMigrationQueue() {
+    return "migration/create-record";
+  }
+
+  @GetMapping(value = "migration/move-file")
+  public String migrateFile() {
+    return "migration/move-file";
+  }
+
+  @GetMapping(value = "migration/sync-new-url")
+  public String syncNewUrl() {
+    return "migration/sync-new-url";
+  }
+}

+ 3 - 0
oos-migration/src/main/resources/OOSCredentials.properties

@@ -0,0 +1,3 @@
+## OOS api key
+accessKey=4d39248aaa33fab47363
+secretKey=344b89df09e717ba29f6487f6f472438a7a7e217

+ 22 - 0
oos-migration/src/main/resources/application-dev.yml

@@ -0,0 +1,22 @@
+
+spring:
+  ####
+  # DataSource
+  ##
+  datasource:
+    driver-class-name: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://localhost:3306/file_migration?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf-8
+    username: root
+    password: 123456
+  jpa:
+    database: mysql
+    show-sql: false
+    hibernate:
+      ddl-auto: update
+    properties:
+      dialect: org.hibernate.dialect.MySQL5Dialect
+  ####
+  # Template Engine
+  ##
+  thymeleaf:
+    cache: false

+ 22 - 0
oos-migration/src/main/resources/application-test.yml

@@ -0,0 +1,22 @@
+
+spring:
+  ####
+  # DataSource
+  ##
+  datasource:
+    driver-class-name: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://192.168.253.12:3306/mall_test?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf-8
+    username: root
+    password: select111***
+  jpa:
+    database: mysql
+    show-sql: false
+    hibernate:
+      ddl-auto: update
+    properties:
+      dialect: org.hibernate.dialect.MySQL5Dialect
+  ####
+  # Template Engine
+  ##
+  thymeleaf:
+    cache: false

+ 12 - 0
oos-migration/src/main/resources/application.yml

@@ -0,0 +1,12 @@
+server:
+  port: 20290
+
+spring:
+  application:
+    name: oos-dashboard
+  profiles:
+    active: test
+
+logging:
+  level:
+    com.uas.platform.oos: debug

+ 32 - 0
oos-migration/src/main/resources/templates/index.html

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8"/>
+  <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
+  <title>Home(Dashboard)</title>
+  <meta name="description" content=""/>
+  <meta name="viewport" content="width=device-width"/>
+  <base href="/"/>
+  <link rel="stylesheet" type="text/css"
+        href="/webjars/bootstrap/css/bootstrap.min.css"/>
+</head>
+<body>
+<div class="container">
+  <h1>Home</h1>
+  <div class="row">
+    <!-- file migration -->
+    <div class="col-md-6 col-sm-12">
+      <h3>文件迁移</h3>
+      <ol>
+        <li><a href="migration/create-record">建立文件迁移记录</a></li>
+        <li><a href="migration/move-file">迁移文件到OOS</a></li>
+        <li><a href="migration/sync-new-url">同步数据库中文件链接</a></li>
+      </ol>
+    </div>
+  </div>
+</div>
+
+<script type="text/javascript" src="/webjars/jquery/jquery.min.js"></script>
+<script type="text/javascript" src="/webjars/bootstrap/js/bootstrap.min.js"></script>
+</body>
+</html>

+ 135 - 0
oos-migration/src/main/resources/templates/migration/create-record.html

@@ -0,0 +1,135 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8"/>
+  <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
+  <title>CreateRecord(Dashboard)</title>
+  <meta name="description" content=""/>
+  <meta name="viewport" content="width=device-width"/>
+  <base href="/"/>
+  <link rel="stylesheet" type="text/css"
+        href="/webjars/bootstrap/css/bootstrap.min.css"/>
+</head>
+<body>
+<div class="container" ng-app="app">
+  <h1>建立文件迁移记录
+    <small><a href="index">首页</a></small>
+  </h1>
+  <div style="margin-top: 40px;margin-bottom: 40px;"
+       ng-controller="MigrationController as migration">
+    <div class="row">
+      <div class="col-sm-6">
+        <form>
+          <div class="form-group" style="margin-top: 20px; margin-bottom: 20px;">
+            <label for="tableName">数据库表名</label>
+            <input class="form-control" id="tableName" name="tableName"
+                   placeholder="Table Name, eg. store$info"
+                   ng-model="migration.tableName"/>
+          </div>
+          <div class="form-group" style="margin-top: 20px; margin-bottom: 20px;">
+            <label for="fieldName">资源文件字段名</label>
+            <input class="form-control" id="fieldName" name="fieldName"
+                   placeholder="Field Name, eg. ST_LOGO_URL"
+                   ng-model="migration.fieldName"/>
+          </div>
+          <div class="form-group" ng-if="migration.count > 0">
+            <span ng-if="migration.statistic">
+              总资源链接数: <span ng-bind="migration.statistic.total"></span>&nbsp;&nbsp;
+              待迁移链接数: <span ng-bind="migration.statistic.todo"></span>&nbsp;&nbsp;
+            </span>
+            共花费时间: <span ng-bind="migration.count / 1000"></span> s
+          </div>
+          <button class="btn btn-default" ng-click="migration.createMigrationRecords()">创建</button>
+        </form>
+      </div>
+      <div class="col-sm-6">
+      </div>
+    </div>
+  </div>
+  <div class="row">
+    <div class="col-sm-offset-3 col-sm-3" style="text-align: right;">
+      <a href="migration/move-file">Next: 迁移文件到OOS</a>
+    </div>
+    <div class="col-sm-6"></div>
+  </div>
+
+</div>
+
+<!-- Scripts -->
+<script type="text/javascript" src="/webjars/jquery/jquery.min.js"></script>
+<script type="text/javascript" src="/webjars/bootstrap/js/bootstrap.min.js"></script>
+<script type="text/javascript" src="/webjars/angularjs/angular.min.js"></script>
+<script type="text/javascript">
+  angular.module("app", [])
+  .config(function ($httpProvider) {
+    $httpProvider.defaults.headers.common['X-Requested-With'] = 'XMLHttpRequest';
+  }).controller("MigrationController", function ($http) {
+    const self = this;
+
+    self.tableName = undefined;
+    self.fieldName = undefined;
+
+    self.statistic = null;
+
+    self.count = 0;
+
+    self.createMigrationRecords = function () {
+      if (!self.tableName || self.tableName === '') {
+        alert('数据库表名不能为空');
+        return ;
+      }
+      if (!self.fieldName || self.fieldName === '') {
+        alert('资源文件字段名不能为空');
+        return ;
+      }
+
+      const params = { tableName: self.tableName, fieldName: self.fieldName };
+
+      $http({
+        method: 'POST',
+        url: 'api/file/migration//create-records',
+        params: params,
+        transformRequest: angular.identity
+      }).success(function (response) {
+        console.log(response);
+        if (response.success) {
+          self.count = response.data;
+
+          $http({
+            method: 'GET',
+            url: 'api/file/migration//migrate-statistic',
+            params: params,
+            transformRequest: angular.identity
+          }).success(function (response) {
+            console.log(response);
+            self.statistic = response.data;
+          }).error(function (error) {
+            console.log(error);
+
+            self.statistic = { total: 0, todo: 0 };
+
+            if (response.status) {
+              console.log(response.status + ":" + response.message);
+            } else {
+              console.log(response.message);
+            }
+          });
+          alert('操作成功');
+        } else {
+          self.errorMessage = response.message;
+        }
+      }).error(function (response) {
+        console.log(response);
+        if (response.status) {
+          alert(response.status + ":" + response.message);
+        } else {
+          alert(response.message);
+        }
+      })
+    };
+
+  });
+</script>
+
+</body>
+</html>

+ 132 - 0
oos-migration/src/main/resources/templates/migration/move-file.html

@@ -0,0 +1,132 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8"/>
+  <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
+  <title>MoveFile(Dashboard)</title>
+  <meta name="description" content=""/>
+  <meta name="viewport" content="width=device-width"/>
+  <base href="/"/>
+  <link rel="stylesheet" type="text/css"
+        href="/webjars/bootstrap/css/bootstrap.min.css"/>
+</head>
+<body>
+<div class="container" ng-app="app">
+  <h1>迁移文件到OOS
+    <small><a href="index">首页</a></small>
+  </h1>
+  <div style="margin-top: 40px;margin-bottom: 40px;"
+       ng-controller="MigrationController as migration">
+    <div class="row">
+      <div class="col-sm-6">
+        <table class="table table-bordered">
+          <tr>
+            <th class="text-center">字段名</th>
+            <th class="text-center">状态</th>
+            <th class="text-center">总数</th>
+          </tr>
+          <tr ng-if="!migration.statistic || migration.statistic.length == 0">
+            <td class="text-center" colspan="3">没有获取到数据,请刷新页面</td>
+          </tr>
+          <tr ng-if="migration.statistic.length > 0"
+              ng-repeat="data in migration.statistic">
+            <td class="text-center" ng-bind="data.fieldName"></td>
+            <td class="text-center" ng-bind="data.status"></td>
+            <td class="text-center" ng-bind="data.total"></td>
+          </tr>
+        </table>
+        <form>
+          <div class="form-group" ng-if="migration.count > 0">
+            共花费时间: <span ng-bind="migration.count / 1000"></span> s
+          </div>
+          <button class="btn btn-default" ng-click="migration.moveFileToServer()">迁移 200 个</button>
+        </form>
+      </div>
+      <div class="col-sm-6">
+      </div>
+    </div>
+  </div>
+  <div class="row">
+    <div class="col-sm-3">
+      <a href="migration/create-record">Pre: 建立文件迁移记录</a>
+    </div>
+    <div class="col-sm-3" style="text-align: right;">
+      <a href="migration/sync-new-url">Next: 同步数据库中文件链接</a>
+    </div>
+    <div class="col-sm-6"></div>
+  </div>
+
+</div>
+
+<!-- Scripts -->
+<script type="text/javascript" src="/webjars/jquery/jquery.min.js"></script>
+<script type="text/javascript" src="/webjars/bootstrap/js/bootstrap.min.js"></script>
+<script type="text/javascript" src="/webjars/angularjs/angular.min.js"></script>
+<script type="text/javascript">
+  angular.module("app", [])
+  .config(function ($httpProvider) {
+    $httpProvider.defaults.headers.common['X-Requested-With'] = 'XMLHttpRequest';
+  }).controller("MigrationController", function ($http) {
+    const self = this;
+
+    self.statistic = [];
+
+    self.count = 0;
+
+    loadStatistic();
+
+    self.moveFileToServer = function () {
+
+      $http({
+        method: 'POST',
+        url: 'api/file/migration//move-file',
+        transformRequest: angular.identity
+      }).success(function (response) {
+        console.log(response);
+        if (response.success) {
+          self.count = response.data;
+          loadStatistic();
+
+          alert('操作成功');
+        } else {
+          self.errorMessage = response.message;
+        }
+      }).error(function (response) {
+        console.log(response);
+        if (response.status) {
+          alert(response.status + ":" + response.message);
+        } else {
+          alert(response.message);
+        }
+      })
+    };
+
+    function loadStatistic() {
+      $http({
+        method: 'GET',
+        url: 'api/file/migration//migration-statistic',
+        transformRequest: angular.identity
+      }).success(function (response) {
+        console.log(response);
+        if (response.success) {
+          self.statistic = response.data;
+        } else {
+          self.statistic = [];
+        }
+      }).error(function (response) {
+        console.log(response);
+
+        self.statistic = [];
+        if (response.status) {
+          console.log(response.status + ":" + response.message);
+        } else {
+          console.log(response.message);
+        }
+      })
+    }
+
+  });
+</script>
+
+</body>
+</html>

+ 137 - 0
oos-migration/src/main/resources/templates/migration/sync-new-url.html

@@ -0,0 +1,137 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8"/>
+  <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
+  <title>MoveFile(Dashboard)</title>
+  <meta name="description" content=""/>
+  <meta name="viewport" content="width=device-width"/>
+  <base href="/"/>
+  <link rel="stylesheet" type="text/css"
+        href="/webjars/bootstrap/css/bootstrap.min.css"/>
+</head>
+<body>
+<div class="container" ng-app="app">
+  <h1>同步数据库中文件链接
+    <small><a href="index">首页</a></small>
+  </h1>
+  <div style="margin-top: 40px;margin-bottom: 40px;"
+       ng-controller="MigrationController as migration">
+    <div class="row">
+      <div class="col-sm-6">
+        <form>
+          <div class="form-group">
+            <label>同步字段</label>
+            <select class="form-control" title="field_full_name" ng-model="migration.selectedField">
+              <option value="N/A">请选择同步字段</option>
+              <option ng-repeat="field in migration.fieldList"
+                      ng-value="field"
+                      ng-bind="field">
+              </option>
+            </select>
+          </div>
+          <div class="form-group" ng-if="migration.count > 0">
+            共花费时间: <span ng-bind="migration.count / 1000"></span> s
+          </div>
+          <button class="btn btn-default" ng-click="migration.syncResourceLink()">同步链接</button>
+        </form>
+      </div>
+      <div class="col-sm-6">
+      </div>
+    </div>
+  </div>
+  <div class="row">
+    <div class="col-sm-3">
+      <a href="migration/move-file">Pre: 迁移文件到OOS</a>
+    </div>
+    <div class="col-sm-3" style="text-align: right;"></div>
+    <div class="col-sm-6"></div>
+  </div>
+
+</div>
+
+<!-- Scripts -->
+<script type="text/javascript" src="/webjars/jquery/jquery.min.js"></script>
+<script type="text/javascript" src="/webjars/bootstrap/js/bootstrap.min.js"></script>
+<script type="text/javascript" src="/webjars/angularjs/angular.min.js"></script>
+<script type="text/javascript">
+  angular.module("app", [])
+  .config(function ($httpProvider) {
+    $httpProvider.defaults.headers.common['X-Requested-With'] = 'XMLHttpRequest';
+  }).controller("MigrationController", function ($http) {
+    const self = this;
+
+    self.fieldList = [];
+    self.selectedField = 'N/A';
+
+    self.count = 0;
+
+    gainFields();
+
+    self.syncResourceLink = function () {
+      console.log(self.selectedField);
+
+      if (!self.selectedField || self.selectedField === '' || self.selectedField === 'N/A') {
+        alert('请选择需要同步的字段名称');
+        return ;
+      }
+
+      const names = self.selectedField.split('.');
+      const params = { tableName: names[0], fieldName: names[1] };
+      console.log(params);
+
+      $http({
+        method: 'PUT',
+        url: 'api/file/migration//sync-links',
+        params: params,
+        transformRequest: angular.identity
+      }).success(function (response) {
+        console.log(response);
+        if (response.success) {
+          self.count = response.data;
+          gainFields();
+
+          alert('操作成功');
+        } else {
+          alert(response.message);
+        }
+      }).error(function (response) {
+        console.log(response);
+
+        if (response.status) {
+          alert(response.status + ":" + response.message);
+        } else {
+          alert(response.message);
+        }
+      })
+    };
+
+    function gainFields() {
+      $http({
+        method: 'GET',
+        url: 'api/file/migration//gain-fields',
+        transformRequest: angular.identity
+      }).success(function (response) {
+        console.log(response);
+        if (response.success) {
+          self.fieldList = response.data;
+        } else {
+          self.fieldList = [];
+        }
+      }).error(function (response) {
+        console.log(response);
+
+        self.fieldList = [];
+        if (response.status) {
+          console.log(response.status + ":" + response.message);
+        } else {
+          console.log(response.message);
+        }
+      });
+    }
+
+  });
+</script>
+
+</body>
+</html>

+ 4 - 0
settings.gradle

@@ -0,0 +1,4 @@
+rootProject.name = 'oos-parent'
+include 'oos-dashboard'
+include 'oos-core'
+include 'oos-migration'