>

Azure의 블록 Blob 저장소에 문제가 있습니다. 문제는 저장소에 액세스하고 항아리에서 컨테이너를 만들려고하면 제대로 작동한다는 것입니다. spark-submit 명령으로 실행하려고하면 작동하지 않습니다. 코드와 Azure 간의 트래픽을 캡처하여 문제가 발생하는 위치를 확인하려고하지만 문제는 www.google.com과 같은 다른 사이트에 액세스 할 때 트래픽을 캡처 할 수는 있지만 피들러가 이러한 종류의 트래픽을 캡처하지 못한다는 것입니다.

이 작동합니다 :

import java.net.*;
import java.io.*;
public class Example
{
    public static void main(String[] args) throws Exception
    {
        System.setProperty("proxySet", "true");
        System.setProperty("proxyHost", "127.0.0.1");
        System.setProperty("proxyPort", "9090");
        System.setProperty("javax.net.ssl.trustStore", "C:\\data\\keys\\FiddlerKeystore");
        System.setProperty("javax.net.ssl.trustStorePassword", "password");

        URL x = new URL("https://www.google.com");
        HttpURLConnection hc = (HttpURLConnection)x.openConnection();
        hc.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2");
        InputStream is = hc.getInputStream();
        int u = 0;
        byte[] kj = new byte[1024];
        while((u = is.read(kj)) != -1)
        {
            System.out.write(kj,0,u);
        }
        is.close();
    }
}

이제 Azure 코드로 동일하게 수행하면 Fiddler는 아무것도 캡처하지 않습니다. 여기 내 Azure 코드가 있습니다 :

import azure.AzureBlockBlobClient;
import common.AzureConf;
import org.apache.log4j.BasicConfigurator;
import java.io.IOException;

public class AzureExample {
    private AzureBlockBlobClient azureBlockBlobClient;
    private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(AzureExample.class);

    public AzureExample() {
        azureBlockBlobClient = new AzureBlockBlobClient(AzureConf.ACCOUNT_NAME,AzureConf.ACCOUNT_KEY, AzureConf.CONTAINER_NAME);
        azureBlockBlobClient.createContainer();
    }

    public static void main(String... args) throws IOException {
        BasicConfigurator.configure();
        System.setProperty("proxySet", "true");
        System.setProperty("proxyHost", "127.0.0.1");
        System.setProperty("proxyPort", "9090");
        System.setProperty("javax.net.ssl.trustStore", "C:\\data\\keys\\FiddlerKeystore");
        System.setProperty("javax.net.ssl.trustStorePassword", "password");
        new AzureExample();
        System.exit(0);
    }
}

다음은 Azure에 연결하는 클라이언트입니다.

   public AzureBlockBlobClient(String accountName, String accountKey, String containerName) {
    this.accountName = accountName;
    this.accountKey = accountKey;
    this.containerName = containerName;
    init();
}
private void init() {
    log.info("Init AzureBlockBlobClient started...");
    try {
        SharedKeyCredentials creds = new SharedKeyCredentials(accountName, accountKey);
        serviceURL = new ServiceURL(new URL("https://" + accountName + ".blob.core.windows.net/"),
            StorageURL.createPipeline(creds, new PipelineOptions()));
        containerURL = serviceURL.createContainerURL(containerName);
    }catch (InvalidKeyException e){
        log.error("Authentication error while trying to access storage account", e);
    }catch (MalformedURLException e) {
        log.error("Invalid Service URL", e);
        e.printStackTrace();
    }catch (Exception e) {
        e.printStackTrace();
        log.error("Error initializing AzureBlockBlobClient", e);
    }
    log.info("Init AzureBlockBlobClient Done!");
}
public void createContainer(){
    try {
        // Let's create a container using a blocking call to Azure Storage
        // If container exists, we'll catch and continue
        log.info("Creating container {}." , containerName);
        ContainerCreateResponse response = containerURL.create(null, null, null).blockingGet();
        log.info("Container Create Response was {}." , response.statusCode());
    }
    catch (RestException e){
        if (e instanceof RestException && e.response().statusCode() != 409) {
            log.error("Error Creating container", e);
        } else {
            log.info("Container {} already exists, resuming...", containerName);
        }
    }
}

여기서 나의 상수가 있습니다 :

public interface AzureConf {
String ACCOUNT_KEY ="<SomeAccountKey>";
String ACCOUNT_NAME = "storage";
String CONTAINER_NAME = "My-container";
}

이것은 나의 maven pom.xml 파일입니다 :

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>examples</groupId>
<artifactId>spark-azure-storage</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
    <junit.version>4.12</junit.version>
</properties>
<dependencies>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-azure</artifactId>
        <version>2.7.1</version>
        <scope>compile</scope>
    </dependency>
    <dependency>
        <groupId>com.microsoft.azure</groupId>
        <artifactId>azure-storage</artifactId>
        <version>2.0.0</version>
        <scope>runtime</scope>
    </dependency>
    <dependency>
        <groupId>com.microsoft.azure</groupId>
        <artifactId>azure-storage-blob</artifactId>
        <version>10.1.0</version>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>${junit.version}</version>
        <scope>runtime</scope>
    </dependency>
    <dependency>
        <groupId>io.reactivex.rxjava2</groupId>
        <artifactId>rxjava</artifactId>
        <version>2.2.3</version>
    </dependency>
    <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-api</artifactId>
        <version>1.7.16</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/com.typesafe.akka/akka-actor -->

    <dependency>
        <groupId>com.microsoft.rest.v2</groupId>
        <artifactId>client-runtime</artifactId>
        <version>2.0.0</version>
        <!--I have to exclude following dependencies and include version 2.9.7 of them otherwise I get
        SoSuchMethodError-->
        <exclusions>
            <exclusion>
                <groupId>com.fasterxml.jackson.core</groupId>
                <artifactId>jackson-annotations</artifactId>
            </exclusion>
            <exclusion>
                <groupId>com.fasterxml.jackson.core</groupId>
                <artifactId>jackson-core</artifactId>
            </exclusion>
            <exclusion>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            </exclusion>
        </exclusions>
    </dependency>
    <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-log4j12</artifactId>
        <version>1.7.16</version>
    </dependency>
    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-core</artifactId>
        <version>2.9.7</version>
    </dependency>
    <dependency>
    <groupId>com.fasterxml.jackson.core</groupId>
    <artifactId>jackson-annotations</artifactId>
    <version>2.9.7</version>
    </dependency>
    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
        <version>2.9.7</version>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_2.11</artifactId>
        <version>2.2.1</version>
    </dependency>
</dependencies>
<build>
    <plugins>
        <plugin>
            <groupId>org.codehaus.mojo</groupId>
            <artifactId>exec-maven-plugin</artifactId>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.8.0</version>
            <configuration>
                <source>1.8</source>
                <target>1.8</target>
            </configuration>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>2.4.3</version>
            <executions>
                <execution>
                    <phase>package</phase>
                    <goals>
                        <goal>shade</goal>
                    </goals>
                </execution>
            </executions>
            <configuration>
                <filters>
                    <filter>
                        <artifact>*:*</artifact>
                        <excludes>
                            <exclude>META-INF/*.SF</exclude>
                            <exclude>META-INF/*.DSA</exclude>
                            <exclude>META-INF/*.RSA</exclude>
                        </excludes>
                    </filter>
                </filters>
                <transformers>
                    <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                        <resource>reference.conf</resource>
                    </transformer>
                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                    <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"></transformer>
                </transformers>
            </configuration>
        </plugin>
    </plugins>
</build>

이 작업을 수행하는 데 도움이 필요하십니까? 미리 감사드립니다


  • 이전 oracle11g - 오라클 견적 도구
  • 다음 java - PostgreSQL은 JPA 및 Hibernate에서 "열은 jsonb 유형이지만 표현식은 bytea 유형"을 발생시킵니다