Apache点燃并引发迭代错误java.lang.NoSuchMethodError：org.apache.spark.sql.SQLContext.createDataFrame

我们想要集成apache ignite和apache spark，我们正在低于错误 .

Maven配置

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.me</groupId>
    <artifactId>igniteclient1</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <packaging>jar</packaging>

    <name>igniteclient1</name>
    <url>http://maven.apache.org</url>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.10 -->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.10</artifactId>
            <version>2.1.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.10 -->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.10</artifactId>
            <version>2.1.0</version>
        </dependency>


        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-core -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-core</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-spring -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-spring</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-indexing -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-indexing</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-rest-http -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-rest-http</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-spark -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-spark</artifactId>
            <version>1.8.0</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>3.8.1</version>
            <scope>test</scope>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.0.0</version>
                <configuration>
                    <filters>
                        <filter>
                            <artifact>*:*</artifact>
                            <excludes>
                                <exclude>META-INF/*.SF</exclude>
                                <exclude>META-INF/*.DSA</exclude>
                                <exclude>META-INF/*.RSA</exclude>
                            </excludes>
                        </filter>
                    </filters>
                    <!-- Additional configuration. -->
                </configuration>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

码：

package com.me.igniteclient1;

import org.apache.spark.api.java.*;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;

import com.me.igniteclient1.ratio.Ratio;

import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.spark.JavaIgniteContext;
import org.apache.ignite.spark.JavaIgniteRDD;
import org.apache.spark.SparkConf;
import org.apache.spark.sql.SQLContext;


/**
 *
 */
public class App 
{
    public static void main(String[] args) {

        SparkConf conf = new SparkConf().setAppName("Stockrank styles");
        JavaSparkContext sc = new JavaSparkContext(conf);

        JavaIgniteContext igniteContext = new JavaIgniteContext(sc, new IgniteConfigProvider());
        JavaIgniteRDD<String, Ratio> javaRdd = igniteContext.fromCache("RatioCache");

        DataFrame df = javaRdd.sql("select ric, ratio1, ratio2, ratio3, ratio4 from Ratio where date = ? and ratios__Exchange = ? and ratio5 > ?", "2017-01-25", "LSE", 10);

        df.printSchema();
        df.show();

        sc.stop();
      }

    /**
     * Ignite configiration provider.
     */
    static class IgniteConfigProvider implements IgniteOutClosure<IgniteConfiguration> {

        public IgniteConfiguration apply() {

            Ignition.setClientMode(true);

            // Start Ignite in client mode.
            Ignite ignite = Ignition.start();
            return ignite.configuration();
        }

    }

}

错误：

Exception in thread "main" java.lang.NoSuchMethodError: org.apache.spark.sql.SQLContext.createDataFrame(Lorg/apache/spark/rdd/RDD;Lorg/apache/spark/sql/types/StructType;)Lorg/apache/spark/sql/DataFrame;
        at org.apache.ignite.spark.IgniteRDD.sql(IgniteRDD.scala:152)
        at org.apache.ignite.spark.JavaIgniteRDD.sql(JavaIgniteRDD.scala:79)
        at org.apache.ignite.spark.JavaIgniteRDD.sql(JavaIgniteRDD.scala:79)
        at com.me.igniteclient1.App.main(App.java:38)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:738)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
    17/02/10 17:55:10 INFO GridUpdateNotifier: Your version is up to date.

这些测试是否通过点燃火花模块？

https://github.com/apache/ignite/blob/master/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java

https://github.com/apache/ignite/blob/master/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java

我尝试了不同的火花版本，1.6,1.5,1.4，但没有任何效果 . 我得到了同样的错误 .

我们正在评估使用点燃（如内存数据结构）和spark（使用mlib进行分析）的可能性 . 但集成模块似乎并不活跃 . 未来的计划是什么？它会被弃用吗？

更新：Ignite 1.9发布https://blogs.apache.org/ignite/entry/apache-ignite-1-9-released

Spark Ignite的spark集成升级到最新的Spark版本 . 目前，您可以在使用最新Spark版本的应用程序中利用Ignite Shared RDD .