我在Scala 2.11.11和Spark 2.2.1上遇到了这个例外

这是最小的例子,代码本身工作正常,但如果我使用ToolBox编译并运行代码,它总是给我 illegal cyclic reference involving object InterfaceAudience 错误 .

package demo

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession

object ToolBoxTest1 extends App {

  val session =
    SparkSession
      .builder()
      .config(
        new SparkConf(true)
          .setAppName("test")
          .setMaster("local")
      )
      .getOrCreate()
  session.sparkContext.parallelize(Range(0, 10))
}

ToolBox版本

package demo

import scala.reflect.runtime.currentMirror
import scala.tools.reflect.ToolBox

object ToolBoxTest2 extends App {
  val code =
    """
      |import org.apache.spark.SparkConf
      |import org.apache.spark.sql.SparkSession
      |val session =
      |  SparkSession
      |    .builder()
      |    .config(
      |      new SparkConf(true)
      |        .setAppName("test")
      |        .setMaster("local")
      |    )
      |    .getOrCreate()
      |session.sparkContext.parallelize(Range(0, 10))
    """.stripMargin
  val tb = currentMirror.mkToolBox()
  tb.compile(tb.parse(code))()
}

线程“main”中的异常scala.tools.reflect.ToolBoxError:反射编译失败:涉及对象InterfaceAudience的非法循环引用,位于scala的scala.tools.reflect.ToolBoxFactory $ ToolBoxImpl $ ToolBoxGlobal.throwIfErrors(ToolBoxFactory.scala:316) . tools.reflect.ToolBoxFactory $ ToolBoxImpl $ ToolBoxGlobal.wrapInPackageAndCompile(ToolBoxFactory.scala:198)at scala.tools.reflect.ToolBoxFactory $ ToolBoxImpl $ ToolBoxGlobal.compile(ToolBoxFactory.scala:252)at scala.tools.reflect.ToolBoxFactory $ ToolBoxImpl $ $ anonfun $ compile $ 2.apply(ToolBoxFactory.scala:429)at scala.tools.reflect.ToolBoxFactory $ ToolBoxImpl $$ anonfun $ compile $ 2.apply(ToolBoxFactory.scala:422)at scala.tools.reflect.ToolBoxFactory $ ToolBoxImpl $ withCompilerApi $ .liftedTree2 $ 1(ToolBoxFactory.scala:355)at scala.tools.reflect.ToolBoxFactory $ ToolBoxImpl $ withCompilerApi $ .apply(ToolBoxFactory.scala:355)at scala.tools.reflect.ToolBoxFactory $ ToolBoxImpl.compile(ToolBoxFactory.scala :422)在demo.ToolBoxTest2 $ .delayedEndpo int $ demo $ ToolBoxTest2 $ 1(ToolBoxTest2.scala:23)at demo.ToolBoxTest2 $ delayedInit $ body.apply(ToolBoxTest2.scala:6)at scala.Function0 $ class.apply $ mcV $ sp(Function0.scala:34)at at scala.runtime.AbstractFunction0.apply $ mcV $ sp(AbstractFunction0.scala:12)at scala.App $$ anonfun $ main $ 1.apply(App.scala:76)at scala.App $$ anonfun $ main $ 1.apply( App.scala:76)scala.collection.immutable.List.foreach(List.scala:392)at scala.collection.generic.TraversableForwarder $ class.foreach(TraversableForwarder.scala:35)at scala.App $ class.main (App.scala:76)at demo.ToolBoxTest2 $ .main(ToolBoxTest2.scala:6)at demo.ToolBoxTest2.main(ToolBoxTest2.scala)

只有在代码中调用parallelize时才会发生这种情况,到目前为止其他所有工作都正常 .

package demo

import scala.reflect.runtime.currentMirror
import scala.tools.reflect.ToolBox

object ToolBoxTest2 extends App {
  val code =
    """
      |import org.apache.spark.SparkConf
      |import org.apache.spark.sql.SparkSession
      |val session =
      |  SparkSession
      |    .builder()
      |    .config(
      |      new SparkConf(true)
      |        .setAppName("test")
      |        .setMaster("local")
      |    )
      |    .getOrCreate()
      |
      |import session.implicits._
      |Range(0, 10).toDF().show()
    """.stripMargin
  val tb = currentMirror.mkToolBox()
  tb.compile(tb.parse(code))()
}

+-----+
|value|
+-----+
|    0|
|    1|
|    2|
|    3|
|    4|
|    5|
|    6|
|    7|
|    8|
|    9|
+-----+