/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.beam.runners.samza.translation;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import org.apache.beam.runners.samza.SamzaExecutionEnvironment;
import org.apache.beam.runners.samza.SamzaPipelineOptions;
import org.apache.beam.runners.samza.SamzaRunner;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.state.StateSpec;
import org.apache.beam.sdk.state.StateSpecs;
import org.apache.beam.sdk.state.ValueState;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Filter;
import org.apache.beam.sdk.transforms.Impulse;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TypeDescriptors;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps;
import org.apache.samza.config.Config;
import org.apache.samza.config.JobConfig;
import org.apache.samza.config.JobCoordinatorConfig;
import org.apache.samza.config.TaskConfig;
import org.apache.samza.config.ZkConfig;
import org.apache.samza.job.yarn.YarnJobFactory;
import org.apache.samza.runtime.LocalApplicationRunner;
import org.apache.samza.runtime.RemoteApplicationRunner;
import org.apache.samza.storage.kv.RocksDbKeyValueStorageEngineFactory;
import org.apache.samza.storage.kv.inmemory.InMemoryKeyValueStorageEngineFactory;
import org.apache.samza.zk.ZkJobCoordinatorFactory;
import org.junit.Test;

/** Test config generations for {@link org.apache.beam.runners.samza.SamzaRunner}. */
// TODO(https://github.com/apache/beam/issues/21230): Remove when new version of errorprone is
// released (2.11.0)
@SuppressWarnings("unused")
public class ConfigGeneratorTest {
  private static final String APP_RUNNER_CLASS = "app.runner.class";
  private static final String JOB_FACTORY_CLASS = "job.factory.class";

  @Test
  public void testStatefulBeamStoreConfig() {
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestStoreConfig");
    options.setRunner(SamzaRunner.class);

    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(Create.of(1, 2, 3)).apply(Sum.integersGlobally());

    pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());

    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final Set<String> nonUniqueStateIds = StateIdParser.scan(pipeline);
    final ConfigContext configCtx = new ConfigContext(idMap, nonUniqueStateIds, options);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    final Config config = configBuilder.build();

    assertEquals(
        RocksDbKeyValueStorageEngineFactory.class.getName(),
        config.get("stores.beamStore.factory"));
    assertEquals("byteArraySerde", config.get("stores.beamStore.key.serde"));
    assertEquals("stateValueSerde", config.get("stores.beamStore.msg.serde"));
    assertNull(config.get("stores.beamStore.changelog"));

    options.setStateDurable(true);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    final Config config2 = configBuilder.build();
    assertEquals(
        "TestStoreConfig-1-beamStore-changelog", config2.get("stores.beamStore.changelog"));
  }

  @Test
  public void testStatelessBeamStoreConfig() {
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestStoreConfig");
    options.setRunner(SamzaRunner.class);

    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(Impulse.create()).apply(Filter.by(Objects::nonNull));

    pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());

    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final Set<String> nonUniqueStateIds = StateIdParser.scan(pipeline);
    final ConfigContext configCtx = new ConfigContext(idMap, nonUniqueStateIds, options);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    final Config config = configBuilder.build();

    assertEquals(
        InMemoryKeyValueStorageEngineFactory.class.getName(),
        config.get("stores.beamStore.factory"));
    assertEquals("byteArraySerde", config.get("stores.beamStore.key.serde"));
    assertEquals("stateValueSerde", config.get("stores.beamStore.msg.serde"));
    assertNull(config.get("stores.beamStore.changelog"));

    options.setStateDurable(true);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    final Config config2 = configBuilder.build();
    // For stateless jobs, ignore state durable pipeline option.
    assertNull(config2.get("stores.beamStore.changelog"));
  }

  @Test
  public void testSamzaLocalExecutionEnvironmentConfig() {
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestEnvConfig");
    options.setRunner(SamzaRunner.class);
    options.setSamzaExecutionEnvironment(SamzaExecutionEnvironment.LOCAL);

    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(Create.of(1, 2, 3)).apply(Sum.integersGlobally());

    pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());

    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final Set<String> nonUniqueStateIds = StateIdParser.scan(pipeline);
    final ConfigContext configCtx = new ConfigContext(idMap, nonUniqueStateIds, options);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    final Config config = configBuilder.build();

    assertTrue(
        Maps.difference(config, ConfigBuilder.localRunConfig()).entriesOnlyOnRight().isEmpty());
  }

  @Test
  public void testSamzaYarnExecutionEnvironmentConfig() {
    final String yarnPackagePath = "yarn.package.path";
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestEnvConfig");
    options.setRunner(SamzaRunner.class);
    options.setSamzaExecutionEnvironment(SamzaExecutionEnvironment.YARN);
    options.setConfigOverride(
        ImmutableMap.<String, String>builder()
            .put(
                yarnPackagePath,
                "file://${basedir}/target/${project.artifactId}-${pom.version}-dist.tar.gz")
            .build());

    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(Create.of(1, 2, 3)).apply(Sum.integersGlobally());

    pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());

    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final Set<String> nonUniqueStateIds = StateIdParser.scan(pipeline);
    final ConfigContext configCtx = new ConfigContext(idMap, nonUniqueStateIds, options);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    try {
      Config config = configBuilder.build();
      assertEquals(config.get(APP_RUNNER_CLASS), RemoteApplicationRunner.class.getName());
      assertEquals(config.get(JOB_FACTORY_CLASS), YarnJobFactory.class.getName());
    } catch (IllegalArgumentException e) {
      throw new AssertionError(
          String.format(
              "Failed to validate correct configs for %s samza execution environment",
              SamzaExecutionEnvironment.YARN),
          e);
    }
  }

  @Test
  public void testSamzaStandAloneExecutionEnvironmentConfig() {
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestEnvConfig");
    options.setRunner(SamzaRunner.class);
    options.setSamzaExecutionEnvironment(SamzaExecutionEnvironment.STANDALONE);
    options.setConfigOverride(
        ImmutableMap.<String, String>builder().put(ZkConfig.ZK_CONNECT, "localhost:2181").build());

    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(Create.of(1, 2, 3)).apply(Sum.integersGlobally());

    pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());

    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final Set<String> nonUniqueStateIds = StateIdParser.scan(pipeline);
    final ConfigContext configCtx = new ConfigContext(idMap, nonUniqueStateIds, options);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    try {
      Config config = configBuilder.build();
      assertEquals(config.get(APP_RUNNER_CLASS), LocalApplicationRunner.class.getName());
      assertEquals(
          config.get(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY),
          ZkJobCoordinatorFactory.class.getName());
    } catch (IllegalArgumentException e) {
      throw new AssertionError(
          String.format(
              "Failed to validate correct configs for %s samza execution environment",
              SamzaExecutionEnvironment.STANDALONE),
          e);
    }
  }

  @Test
  public void testUserStoreConfig() {
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestStoreConfig");
    options.setRunner(SamzaRunner.class);

    Pipeline pipeline = Pipeline.create(options);
    pipeline
        .apply(
            Create.empty(TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.strings())))
        .apply(
            ParDo.of(
                new DoFn<KV<String, String>, Void>() {
                  private static final String testState = "testState";

                  @StateId(testState)
                  private final StateSpec<ValueState<Integer>> state = StateSpecs.value();

                  @ProcessElement
                  public void processElement(
                      ProcessContext context, @StateId(testState) ValueState<Integer> state) {}
                }));

    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final Set<String> nonUniqueStateIds = StateIdParser.scan(pipeline);
    final ConfigContext configCtx = new ConfigContext(idMap, nonUniqueStateIds, options);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);

    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    final Config config = configBuilder.build();

    assertEquals(
        RocksDbKeyValueStorageEngineFactory.class.getName(),
        config.get("stores.testState.factory"));
    assertEquals("byteArraySerde", config.get("stores.testState.key.serde"));
    assertEquals("stateValueSerde", config.get("stores.testState.msg.serde"));
    assertNull(config.get("stores.testState.changelog"));

    options.setStateDurable(true);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    final Config config2 = configBuilder.build();
    assertEquals(
        "TestStoreConfig-1-testState-changelog", config2.get("stores.testState.changelog"));
  }

  @Test
  public void testUserStoreConfigSameStateIdAcrossParDo() {
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestStoreConfig");
    options.setRunner(SamzaRunner.class);

    Pipeline pipeline = Pipeline.create(options);
    pipeline
        .apply(
            Create.empty(TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.strings())))
        .apply(
            "First stateful ParDo",
            ParDo.of(
                new DoFn<KV<String, String>, KV<String, String>>() {
                  private static final String testState = "testState";

                  @StateId(testState)
                  private final StateSpec<ValueState<Integer>> state = StateSpecs.value();

                  @ProcessElement
                  public void processElement(
                      ProcessContext context, @StateId(testState) ValueState<Integer> state) {
                    context.output(context.element());
                  }
                }))
        .apply(
            "Second stateful ParDo",
            ParDo.of(
                new DoFn<KV<String, String>, Void>() {
                  private static final String testState = "testState";

                  @StateId(testState)
                  private final StateSpec<ValueState<Integer>> state = StateSpecs.value();

                  @ProcessElement
                  public void processElement(
                      ProcessContext context, @StateId(testState) ValueState<Integer> state) {}
                }));

    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final Set<String> nonUniqueStateIds = StateIdParser.scan(pipeline);
    final ConfigContext configCtx = new ConfigContext(idMap, nonUniqueStateIds, options);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    final Config config = configBuilder.build();

    assertEquals(
        RocksDbKeyValueStorageEngineFactory.class.getName(),
        config.get("stores.testState-First_stateful_ParDo.factory"));
    assertEquals("byteArraySerde", config.get("stores.testState-First_stateful_ParDo.key.serde"));
    assertEquals("stateValueSerde", config.get("stores.testState-First_stateful_ParDo.msg.serde"));
    assertNull(config.get("stores.testState-First_stateful_ParDo.changelog"));

    assertEquals(
        RocksDbKeyValueStorageEngineFactory.class.getName(),
        config.get("stores.testState-Second_stateful_ParDo.factory"));
    assertEquals("byteArraySerde", config.get("stores.testState-Second_stateful_ParDo.key.serde"));
    assertEquals("stateValueSerde", config.get("stores.testState-Second_stateful_ParDo.msg.serde"));
    assertNull(config.get("stores.testState-Second_stateful_ParDo.changelog"));

    options.setStateDurable(true);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    final Config config2 = configBuilder.build();
    assertEquals(
        "TestStoreConfig-1-testState-First_stateful_ParDo-changelog",
        config2.get("stores.testState-First_stateful_ParDo.changelog"));
    assertEquals(
        "TestStoreConfig-1-testState-Second_stateful_ParDo-changelog",
        config2.get("stores.testState-Second_stateful_ParDo.changelog"));
  }

  @Test
  public void testUserStoreConfigSameStateIdAndPTransformName() {
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestStoreConfig");
    options.setRunner(SamzaRunner.class);

    Pipeline pipeline = Pipeline.create(options);
    pipeline
        .apply(
            Create.empty(TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.strings())))
        .apply(
            "Same stateful ParDo Name",
            ParDo.of(
                new DoFn<KV<String, String>, KV<String, String>>() {
                  private static final String testState = "testState";

                  @StateId(testState)
                  private final StateSpec<ValueState<Integer>> state = StateSpecs.value();

                  @ProcessElement
                  public void processElement(
                      ProcessContext context, @StateId(testState) ValueState<Integer> state) {
                    context.output(context.element());
                  }
                }))
        .apply(
            "Same stateful ParDo Name",
            ParDo.of(
                new DoFn<KV<String, String>, Void>() {
                  private static final String testState = "testState";

                  @StateId(testState)
                  private final StateSpec<ValueState<Integer>> state = StateSpecs.value();

                  @ProcessElement
                  public void processElement(
                      ProcessContext context, @StateId(testState) ValueState<Integer> state) {}
                }));

    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final Set<String> nonUniqueStateIds = StateIdParser.scan(pipeline);
    final ConfigContext configCtx = new ConfigContext(idMap, nonUniqueStateIds, options);

    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    final Config config = configBuilder.build();

    assertEquals(
        RocksDbKeyValueStorageEngineFactory.class.getName(),
        config.get("stores.testState-Same_stateful_ParDo_Name.factory"));
    assertEquals(
        "byteArraySerde", config.get("stores.testState-Same_stateful_ParDo_Name.key.serde"));
    assertEquals(
        "stateValueSerde", config.get("stores.testState-Same_stateful_ParDo_Name.msg.serde"));
    assertNull(config.get("stores.testState-Same_stateful_ParDo_Name.changelog"));

    assertEquals(
        RocksDbKeyValueStorageEngineFactory.class.getName(),
        config.get("stores.testState-Same_stateful_ParDo_Name2.factory"));
    assertEquals(
        "byteArraySerde", config.get("stores.testState-Same_stateful_ParDo_Name2.key.serde"));
    assertEquals(
        "stateValueSerde", config.get("stores.testState-Same_stateful_ParDo_Name2.msg.serde"));
    assertNull(config.get("stores.testState-Same_stateful_ParDo_Name2.changelog"));

    options.setStateDurable(true);
    SamzaPipelineTranslator.createConfig(pipeline, configCtx, configBuilder);
    final Config config2 = configBuilder.build();
    assertEquals(
        "TestStoreConfig-1-testState-Same_stateful_ParDo_Name-changelog",
        config2.get("stores.testState-Same_stateful_ParDo_Name.changelog"));
    assertEquals(
        "TestStoreConfig-1-testState-Same_stateful_ParDo_Name2-changelog",
        config2.get("stores.testState-Same_stateful_ParDo_Name2.changelog"));
  }

  @Test
  public void testCreateBundleConfig() {
    // autosizing = 0: disabled
    // autosizing = 1: enabled
    for (int autosizing = 0; autosizing < 2; autosizing++) {
      final SamzaPipelineOptions options = PipelineOptionsFactory.as(SamzaPipelineOptions.class);
      final Map<String, String> config = new HashMap<>();

      // bundle size = 1
      options.setMaxBundleSize(1);
      config.put(JobConfig.JOB_CONTAINER_THREAD_POOL_SIZE, "5");
      if (autosizing != 0) {
        // Test autosizing enabled, the output should be the same
        config.put(JobConfig.JOB_AUTOSIZING_CONTAINER_THREAD_POOL_SIZE, "5");
      }

      Map<String, String> bundleConfig = ConfigBuilder.createBundleConfig(options, config);

      assertEquals("1", bundleConfig.get(TaskConfig.MAX_CONCURRENCY));
      assertNull(bundleConfig.get(JobConfig.JOB_CONTAINER_THREAD_POOL_SIZE));
      assertNull(bundleConfig.get(JobConfig.JOB_AUTOSIZING_CONTAINER_THREAD_POOL_SIZE));
      assertEquals(1, options.getNumThreadsForProcessElement());

      // bundle size = 3, NumThreadsForProcessElement = 10
      options.setMaxBundleSize(3);
      options.setNumThreadsForProcessElement(10);
      bundleConfig = ConfigBuilder.createBundleConfig(options, config);

      assertEquals("3", bundleConfig.get(TaskConfig.MAX_CONCURRENCY));
      assertEquals("0", bundleConfig.get(JobConfig.JOB_CONTAINER_THREAD_POOL_SIZE));
      assertEquals("0", bundleConfig.get(JobConfig.JOB_AUTOSIZING_CONTAINER_THREAD_POOL_SIZE));
      assertEquals(10, options.getNumThreadsForProcessElement());

      // bundle size = 3, NumThreadsForProcessElement = 1 (default), threadPoolSize = 5
      options.setNumThreadsForProcessElement(1);
      bundleConfig = ConfigBuilder.createBundleConfig(options, config);

      assertEquals("3", bundleConfig.get(TaskConfig.MAX_CONCURRENCY));
      assertEquals("0", bundleConfig.get(JobConfig.JOB_CONTAINER_THREAD_POOL_SIZE));
      assertEquals("0", bundleConfig.get(JobConfig.JOB_AUTOSIZING_CONTAINER_THREAD_POOL_SIZE));
      assertEquals(5, options.getNumThreadsForProcessElement());
    }
  }
}
