Browse Source

fix: aggressively suppress Parquet DEBUG logging

- Set Parquet I/O loggers to OFF (completely disabled)
- Add log4j.configuration system property to ensure config is used
- Override Spark's default log4j configuration
- Prevents thousands of record-level DEBUG messages in CI logs
pull/7526/head
chrislu 7 days ago
parent
commit
150deefdc0
  1. 3
      test/java/spark/pom.xml
  2. 9
      test/java/spark/src/test/resources/log4j.properties

3
test/java/spark/pom.xml

@ -246,6 +246,9 @@
<include>**/*Test.java</include> <include>**/*Test.java</include>
</includes> </includes>
<argLine>${surefire.jvm.args}</argLine> <argLine>${surefire.jvm.args}</argLine>
<systemPropertyVariables>
<log4j.configuration>file:${project.basedir}/src/test/resources/log4j.properties</log4j.configuration>
</systemPropertyVariables>
<environmentVariables> <environmentVariables>
<HADOOP_HOME>/tmp</HADOOP_HOME> <HADOOP_HOME>/tmp</HADOOP_HOME>
</environmentVariables> </environmentVariables>

9
test/java/spark/src/test/resources/log4j.properties

@ -13,10 +13,11 @@ log4j.logger.org.apache.hadoop=WARN
log4j.logger.seaweed=INFO log4j.logger.seaweed=INFO
# Suppress Parquet verbose DEBUG logging # Suppress Parquet verbose DEBUG logging
log4j.logger.org.apache.parquet=WARN
log4j.logger.org.apache.parquet.io=ERROR
log4j.logger.org.apache.parquet.io.RecordConsumerLoggingWrapper=ERROR
log4j.logger.org.apache.parquet.io.MessageColumnIO=ERROR
log4j.logger.org.apache.parquet=ERROR
log4j.logger.org.apache.parquet.io=OFF
log4j.logger.org.apache.parquet.io.RecordConsumerLoggingWrapper=OFF
log4j.logger.org.apache.parquet.io.MessageColumnIO=OFF
log4j.logger.org.apache.parquet.hadoop=ERROR
# Suppress unnecessary warnings # Suppress unnecessary warnings
log4j.logger.org.apache.spark.util.Utils=ERROR log4j.logger.org.apache.spark.util.Utils=ERROR

Loading…
Cancel
Save