/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.beam.sdk.transforms;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;

import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarLongCoder;
import org.apache.beam.sdk.state.TimeDomain;
import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.testing.TestStream;
import org.apache.beam.sdk.testing.UsesTestStream;
import org.apache.beam.sdk.testing.UsesTestStreamWithProcessingTime;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TimestampedValue;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap;
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

@RunWith(JUnit4.class)
public class DeduplicateTest {

  @Rule public TestPipeline p = TestPipeline.create();

  @Test
  @Category({NeedsRunner.class, UsesTestStream.class})
  public void testInDifferentWindows() {
    Instant base = new Instant(0);
    TestStream<String> values =
        TestStream.create(StringUtf8Coder.of())
            .advanceWatermarkTo(base)
            .addElements(
                TimestampedValue.of("k1", base),
                TimestampedValue.of("k2", base.plus(Duration.standardSeconds(10))),
                TimestampedValue.of("k3", base.plus(Duration.standardSeconds(20))),
                TimestampedValue.of("k1", base.plus(Duration.standardSeconds(30))),
                TimestampedValue.of("k2", base.plus(Duration.standardSeconds(40))),
                TimestampedValue.of("k3", base.plus(Duration.standardSeconds(50))),
                TimestampedValue.of("k4", base.plus(Duration.standardSeconds(60))),
                TimestampedValue.of("k5", base.plus(Duration.standardSeconds(70))),
                TimestampedValue.of("k6", base.plus(Duration.standardSeconds(80))))
            .advanceWatermarkToInfinity();

    PCollection<String> distinctValues =
        p.apply(values)
            .apply(Window.into(FixedWindows.of(Duration.standardSeconds(30))))
            .apply(Deduplicate.values());
    PAssert.that(distinctValues)
        .inWindow(new IntervalWindow(base, base.plus(Duration.standardSeconds(30))))
        .containsInAnyOrder("k1", "k2", "k3");
    PAssert.that(distinctValues)
        .inWindow(
            new IntervalWindow(
                base.plus(Duration.standardSeconds(30)), base.plus(Duration.standardSeconds(60))))
        .containsInAnyOrder("k1", "k2", "k3");
    PAssert.that(distinctValues)
        .inWindow(
            new IntervalWindow(
                base.plus(Duration.standardSeconds(60)), base.plus(Duration.standardSeconds(90))))
        .containsInAnyOrder("k4", "k5", "k6");
    p.run();
  }

  @Test
  @Category({NeedsRunner.class, UsesTestStream.class})
  public void testEventTime() {
    Instant base = new Instant(0);
    TestStream<String> values =
        TestStream.create(StringUtf8Coder.of())
            .advanceWatermarkTo(base)
            .addElements(
                TimestampedValue.of("k1", base),
                TimestampedValue.of("k2", base.plus(Duration.standardSeconds(10))),
                TimestampedValue.of("k3", base.plus(Duration.standardSeconds(20))),
                TimestampedValue.of("maybedup", base.plus(Duration.standardSeconds(59))))
            .advanceWatermarkTo(base.plus(Duration.standardMinutes(1)))
            .addElements(
                TimestampedValue.of("k1", base.plus(Duration.standardSeconds(30))),
                TimestampedValue.of("k2", base.plus(Duration.standardSeconds(40))),
                TimestampedValue.of("k3", base.plus(Duration.standardSeconds(50))))
            .advanceWatermarkTo(
                base.plus(Duration.standardMinutes(1)).plus(Deduplicate.DEFAULT_DURATION))
            .addElements(TimestampedValue.of("maybedup", base.plus(Duration.standardSeconds(59))))
            .advanceWatermarkToInfinity();

    PCollection<String> distinctValues =
        p.apply(values).apply(Deduplicate.<String>values().withTimeDomain(TimeDomain.EVENT_TIME));
    PAssert.that(distinctValues)
        .satisfies(
            (Iterable<String> input) -> {
              assertEquals(1, Iterables.frequency(input, "k1"));
              assertEquals(1, Iterables.frequency(input, "k2"));
              assertEquals(1, Iterables.frequency(input, "k3"));
              assertTrue(
                  Iterables.frequency(input, "maybedup") == 1
                      || Iterables.frequency(input, "maybedup") == 2);
              return null;
            });
    p.run();
  }

  @Test
  @Category({NeedsRunner.class, UsesTestStreamWithProcessingTime.class})
  public void testProcessingTime() {
    Instant base = new Instant(0);
    TestStream<String> values =
        TestStream.create(StringUtf8Coder.of())
            .advanceWatermarkTo(base)
            .addElements(
                TimestampedValue.of("k1", base),
                TimestampedValue.of("k2", base.plus(Duration.standardSeconds(10))),
                TimestampedValue.of("k3", base.plus(Duration.standardSeconds(20))),
                TimestampedValue.of("maybedup", base.plus(Duration.standardSeconds(59))))
            .advanceProcessingTime(Duration.standardMinutes(1))
            .addElements(
                TimestampedValue.of("k1", base.plus(Duration.standardSeconds(30))),
                TimestampedValue.of("k2", base.plus(Duration.standardSeconds(40))),
                TimestampedValue.of("k3", base.plus(Duration.standardSeconds(50))))
            .advanceProcessingTime(Deduplicate.DEFAULT_DURATION)
            .addElements(TimestampedValue.of("maybedup", base.plus(Duration.standardSeconds(59))))
            .advanceWatermarkToInfinity();

    PCollection<String> distinctValues = p.apply(values).apply(Deduplicate.values());
    PAssert.that(distinctValues)
        .satisfies(
            (Iterable<String> input) -> {
              assertEquals(1, Iterables.frequency(input, "k1"));
              assertEquals(1, Iterables.frequency(input, "k2"));
              assertEquals(1, Iterables.frequency(input, "k3"));
              assertTrue(
                  Iterables.frequency(input, "maybedup") == 1
                      || Iterables.frequency(input, "maybedup") == 2);
              return null;
            });
    p.run();
  }

  private static class Keys<T> implements SerializableFunction<KV<T, String>, T> {
    @Override
    public T apply(KV<T, String> input) {
      return input.getKey();
    }
  }

  @Test
  @Category({NeedsRunner.class, UsesTestStreamWithProcessingTime.class})
  public void testRepresentativeValuesWithCoder() {
    Instant base = new Instant(0);
    TestStream<KV<Long, String>> values =
        TestStream.create(KvCoder.of(VarLongCoder.of(), StringUtf8Coder.of()))
            .advanceWatermarkTo(base)
            .addElements(
                TimestampedValue.of(KV.of(1L, "k1"), base),
                TimestampedValue.of(KV.of(2L, "k2"), base.plus(Duration.standardSeconds(10))),
                TimestampedValue.of(KV.of(3L, "k3"), base.plus(Duration.standardSeconds(20))))
            .advanceProcessingTime(Duration.standardMinutes(1))
            .addElements(
                TimestampedValue.of(KV.of(1L, "k1"), base.plus(Duration.standardSeconds(30))),
                TimestampedValue.of(KV.of(2L, "k2"), base.plus(Duration.standardSeconds(40))),
                TimestampedValue.of(KV.of(3L, "k3"), base.plus(Duration.standardSeconds(50))))
            .advanceWatermarkToInfinity();

    PCollection<KV<Long, String>> distinctValues =
        p.apply(values)
            .apply(
                Deduplicate.withRepresentativeValueFn(new Keys<Long>())
                    .withRepresentativeCoder(VarLongCoder.of()));

    PAssert.that(distinctValues)
        .containsInAnyOrder(KV.of(1L, "k1"), KV.of(2L, "k2"), KV.of(3L, "k3"));
    p.run();
  }

  @Test
  @Category({NeedsRunner.class, UsesTestStreamWithProcessingTime.class})
  public void testTriggeredRepresentativeValuesWithType() {
    Instant base = new Instant(0);
    TestStream<KV<Long, String>> values =
        TestStream.create(KvCoder.of(VarLongCoder.of(), StringUtf8Coder.of()))
            .advanceWatermarkTo(base)
            .addElements(
                TimestampedValue.of(KV.of(1L, "k1"), base),
                TimestampedValue.of(KV.of(2L, "k2"), base.plus(Duration.standardSeconds(10))),
                TimestampedValue.of(KV.of(3L, "k3"), base.plus(Duration.standardSeconds(20))))
            .advanceProcessingTime(Duration.standardMinutes(1))
            .addElements(
                TimestampedValue.of(KV.of(1L, "k1"), base.plus(Duration.standardSeconds(30))),
                TimestampedValue.of(KV.of(2L, "k2"), base.plus(Duration.standardSeconds(40))),
                TimestampedValue.of(KV.of(3L, "k3"), base.plus(Duration.standardSeconds(50))))
            .advanceWatermarkToInfinity();

    PCollection<KV<Long, String>> distinctValues =
        p.apply(values)
            .apply(
                Deduplicate.withRepresentativeValueFn(new Keys<Long>())
                    .withRepresentativeCoder(VarLongCoder.of()));

    PAssert.that(distinctValues)
        .containsInAnyOrder(KV.of(1L, "k1"), KV.of(2L, "k2"), KV.of(3L, "k3"));
    p.run();
  }

  @Test
  public void withLambdaRepresentativeValuesFnNoTypeDescriptorShouldThrow() {

    Multimap<Integer, String> predupedContents = HashMultimap.create();
    predupedContents.put(3, "foo");
    predupedContents.put(4, "foos");
    predupedContents.put(6, "barbaz");
    predupedContents.put(6, "bazbar");
    PCollection<String> dupes =
        p.apply(Create.of("foo", "foos", "barbaz", "barbaz", "bazbar", "foo"));

    assertThrows(
        "Unable to return a default Coder for RemoveRepresentativeDupes",
        IllegalStateException.class,
        () ->
            dupes.apply(
                "RemoveRepresentativeDupes",
                Deduplicate.withRepresentativeValueFn(String::length)));
  }
}
