Twitter functional apps
* Streaming source * Add twitter search and messasge sources * Add more twitter source, sink and processors * Add IT tests for twitter update sink * Add IT tests for twitter message sink * Add IT tests for twitter trend processor * twitter suppliers readme * twitter consumers readme * twitter functions readme * Disable TwitterStreamSourceTests
This commit is contained in:
Christian Tzolov
committed by
Soby Chacko
Soby Chacko
parent
9bf8dcc4e7
commit
873f17414e
43
applications/source/twitter-stream-source/README.adoc
Normal file
43
applications/source/twitter-stream-source/README.adoc
Normal file
@@ -0,0 +1,43 @@
|
||||
//tag::ref-doc[]
|
||||
= Twitter Stream Source
|
||||
|
||||
Real-time Tweet streaming https://developer.twitter.com/en/docs/tweets/filter-realtime/api-reference/post-statuses-filter.html[Filter] and https://developer.twitter.com/en/docs/tweets/sample-realtime/overview/GET_statuse_sample[Sample] APIs support.
|
||||
|
||||
* The `Filter API` returns public statuses that match one or more filter predicates.
|
||||
Multiple parameters allows using a single connection to the Streaming API.
|
||||
TIP: The `track`, `follow`, and `locations` fields are combined with an *OR* operator!
|
||||
Queries with `track=foo` and `follow=1234` returns Tweets matching `foo` *OR* created by user `1234`.
|
||||
|
||||
* The `Sample API` returns a small random sample of all public statuses.
|
||||
The Tweets returned by the default access level are the same, so if two different clients connect to this endpoint, they will see the same Tweets.
|
||||
|
||||
The default access level allows up to 400 track keywords, 5,000 follow user Ids and 25 0.1-360 degree location boxes.
|
||||
|
||||
== Options
|
||||
|
||||
//tag::configuration-properties[]
|
||||
$$twitter.connection.access-token$$:: $$Your Twitter token.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$twitter.connection.access-token-secret$$:: $$Your Twitter token secret.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$twitter.connection.consumer-key$$:: $$Your Twitter key.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$twitter.connection.consumer-secret$$:: $$Your Twitter secret.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$twitter.connection.debug-enabled$$:: $$Enables Twitter4J debug mode.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$twitter.connection.raw-json$$:: $$Enable caching the original (raw) JSON objects as returned by the Twitter APIs. When set to False the result will use the Twitter4J's json representations. When set to True the result will use the original Twitter APISs json representations.$$ *($$Boolean$$, default: `$$true$$`)*
|
||||
$$twitter.stream.filter.count$$:: $$Indicates the number of previous statuses to stream before transitioning to the live stream.$$ *($$Integer$$, default: `$$0$$`)*
|
||||
$$twitter.stream.filter.filter-level$$:: $$The filter level limits what tweets appear in the stream to those with a minimum filterLevel attribute value. One of either none, low, or medium.$$ *($$FilterLevel$$, default: `$$<none>$$`)*
|
||||
$$twitter.stream.filter.follow$$:: $$Specifies the users, by ID, to receive public tweets from.$$ *($$List<Long>$$, default: `$$<none>$$`)*
|
||||
$$twitter.stream.filter.language$$:: $$Specifies the tweets language of the stream.$$ *($$List<String>$$, default: `$$<none>$$`)*
|
||||
$$twitter.stream.filter.locations$$:: $$Locations to track. Internally represented as 2D array. Bounding box is invalid: 52.38, 4.90, 51.51, -0.12. The first pair must be the SW corner of the box$$ *($$List<BoundingBox>$$, default: `$$<none>$$`)*
|
||||
$$twitter.stream.filter.track$$:: $$Specifies keywords to track.$$ *($$List<String>$$, default: `$$<none>$$`)*
|
||||
$$twitter.stream.type$$:: $$<documentation missing>$$ *($$StreamType$$, default: `$$<none>$$`, possible values: `sample`,`filter`,`firehose`,`link`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
//end::ref-doc[]
|
||||
|
||||
|
||||
== Examples
|
||||
|
||||
```
|
||||
java -jar twitter-stream-source.jar --twitter.connection.accessTokenSecret=xxx --twitter.connection.accessToken=xxx --twitter.connection.consumerKey=xxx --twitter.connection.consumerSecret=xxx
|
||||
```
|
||||
|
||||
|
||||
93
applications/source/twitter-stream-source/pom.xml
Normal file
93
applications/source/twitter-stream-source/pom.xml
Normal file
@@ -0,0 +1,93 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>twitter-stream-source</artifactId>
|
||||
<version>3.0.0-SNAPSHOT</version>
|
||||
<name>twitter-stream-source</name>
|
||||
<description>twitter stream source apps</description>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<parent>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
<artifactId>stream-applications-core</artifactId>
|
||||
<version>3.0.0-SNAPSHOT</version>
|
||||
<relativePath/>
|
||||
</parent>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>twitter-supplier</artifactId>
|
||||
<version>${java-functions.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>function-test-support</artifactId>
|
||||
<version>${java-functions.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.mock-server</groupId>
|
||||
<artifactId>mockserver-netty</artifactId>
|
||||
<version>5.10</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mock-server</groupId>
|
||||
<artifactId>mockserver-client-java</artifactId>
|
||||
<version>5.10</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-app-starter-doc-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.cloud.stream.app.plugin</groupId>
|
||||
<artifactId>spring-cloud-stream-app-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<generatedApp>
|
||||
<name>twitter-stream</name>
|
||||
<type>source</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.supplier.twitter.status.stream.TwitterStreamSupplierConfiguration.class</configClass>
|
||||
<functionDefinition>twitterStreamSupplier</functionDefinition>
|
||||
</generatedApp>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>twitter-supplier</artifactId>
|
||||
<version>${java-functions.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
<id>spring-snapshots</id>
|
||||
<name>Spring Snapshots</name>
|
||||
<url>https://repo.spring.io/libs-snapshot-local</url>
|
||||
</repository>
|
||||
<repository>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring Milestones</name>
|
||||
<url>https://repo.spring.io/libs-milestone-local</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,5 @@
|
||||
configuration-properties.classes=org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties,\
|
||||
org.springframework.cloud.fn.supplier.twitter.status.stream.TwitterStreamSupplierProperties, \
|
||||
org.springframework.cloud.fn.supplier.twitter.status.stream.TwitterStreamSupplierProperties$Filter, \
|
||||
org.springframework.cloud.fn.supplier.twitter.status.stream.TwitterStreamSupplierProperties$Filter$BoundingBox, \
|
||||
org.springframework.cloud.fn.supplier.twitter.status.stream.TwitterStreamSupplierProperties$Filter$Geocode
|
||||
@@ -0,0 +1,164 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.app.source.twitter.stream;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockserver.client.MockServerClient;
|
||||
import org.mockserver.integration.ClientAndServer;
|
||||
import org.mockserver.model.Header;
|
||||
import org.mockserver.model.HttpRequest;
|
||||
import org.mockserver.model.StringBody;
|
||||
import twitter4j.conf.ConfigurationBuilder;
|
||||
|
||||
import org.springframework.boot.SpringBootConfiguration;
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties;
|
||||
import org.springframework.cloud.fn.supplier.twitter.status.stream.TwitterStreamSupplierConfiguration;
|
||||
import org.springframework.cloud.fn.supplier.twitter.status.stream.TwitterStreamSupplierProperties;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockserver.matchers.Times.exactly;
|
||||
import static org.mockserver.model.HttpRequest.request;
|
||||
import static org.mockserver.model.HttpResponse.response;
|
||||
import static org.mockserver.verify.VerificationTimes.once;
|
||||
|
||||
public class TwitterStreamSourceTests {
|
||||
|
||||
private static final String MOCK_SERVER_IP = "127.0.0.1";
|
||||
|
||||
private static final Integer MOCK_SERVER_PORT = 1080;
|
||||
|
||||
private static ClientAndServer mockServer;
|
||||
|
||||
private static MockServerClient mockClient;
|
||||
private static HttpRequest streamFilterRequest;
|
||||
private static HttpRequest streamSampleRequest;
|
||||
private static HttpRequest streamFirehoseRequest;
|
||||
private static HttpRequest streamLinknsRequest;
|
||||
|
||||
@BeforeAll
|
||||
public static void startServer() {
|
||||
|
||||
mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT);
|
||||
|
||||
mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT);
|
||||
|
||||
streamFilterRequest = mockClientRecordRequest(request()
|
||||
.withMethod("POST")
|
||||
.withPath("/stream/statuses/filter.json")
|
||||
.withBody(new StringBody("count=0&track=Java%2CPython&stall_warnings=true")));
|
||||
|
||||
streamSampleRequest = mockClientRecordRequest(request()
|
||||
.withMethod("GET")
|
||||
.withPath("/stream/statuses/sample.json"));
|
||||
|
||||
streamLinknsRequest = mockClientRecordRequest(request()
|
||||
.withMethod("GET")
|
||||
.withPath("/stream/statuses/links.json"));
|
||||
//.withBody(new StringBody("count=0&stall_warnings=true")));
|
||||
|
||||
streamFirehoseRequest = mockClientRecordRequest(request()
|
||||
.withMethod("POST")
|
||||
.withPath("/stream/statuses/firehose.json")
|
||||
.withBody(new StringBody("count=0&stall_warnings=true")));
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
public static void stopServer() {
|
||||
mockServer.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
@Disabled
|
||||
public void testSourceFromSupplier() {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
|
||||
TestChannelBinderConfiguration.getCompleteConfiguration(TestTwitterStreamSourceApplication.class))
|
||||
.web(WebApplicationType.NONE)
|
||||
.run("--spring.cloud.stream.function.definition=twitterStreamSupplier",
|
||||
|
||||
"--twitter.connection.consumerKey=consumerKey666",
|
||||
"--twitter.connection.consumerSecret=consumerSecret666",
|
||||
"--twitter.connection.accessToken=accessToken666",
|
||||
"--twitter.connection.accessTokenSecret=accessTokenSecret666",
|
||||
|
||||
"--twitter.stream.type=filter",
|
||||
"--twitter.stream.filter.track=Java,Python",
|
||||
"--twitter.stream.filter.count=3")) {
|
||||
|
||||
TwitterConnectionProperties twitterConnectionProperties = context.getBean(TwitterConnectionProperties.class);
|
||||
assertThat(twitterConnectionProperties.getConsumerKey()).isEqualTo("consumerKey666");
|
||||
assertThat(twitterConnectionProperties.getConsumerSecret()).isEqualTo("consumerSecret666");
|
||||
assertThat(twitterConnectionProperties.getAccessToken()).isEqualTo("accessToken666");
|
||||
assertThat(twitterConnectionProperties.getAccessTokenSecret()).isEqualTo("accessTokenSecret666");
|
||||
|
||||
TwitterStreamSupplierProperties twitterStreamSupplierProperties =
|
||||
context.getBean(TwitterStreamSupplierProperties.class);
|
||||
assertThat(twitterStreamSupplierProperties.getType()).isEqualTo(TwitterStreamSupplierProperties.StreamType.filter);
|
||||
assertThat(twitterStreamSupplierProperties.getFilter().getTrack()).contains("Java", "Python");
|
||||
|
||||
//OutputDestination target = context.getBean(OutputDestination.class);
|
||||
//Message<byte[]> sourceMessage = target.receive(10000);
|
||||
//final String actual = new String(sourceMessage.getPayload());
|
||||
|
||||
mockClient.verify(streamFilterRequest, once());
|
||||
}
|
||||
}
|
||||
|
||||
private static HttpRequest mockClientRecordRequest(HttpRequest request) {
|
||||
mockClient.when(request, exactly(1))
|
||||
.respond(response()
|
||||
.withStatusCode(200)
|
||||
.withHeaders(
|
||||
new Header("Content-Type", "application/json; charset=utf-8"),
|
||||
new Header("Cache-Control", "public, max-age=86400"))
|
||||
.withBody(TwitterTestUtils.asString("classpath:/response/stream_test_1.json"))
|
||||
.withDelay(TimeUnit.SECONDS, 10));
|
||||
return request;
|
||||
}
|
||||
|
||||
@SpringBootConfiguration
|
||||
@EnableAutoConfiguration
|
||||
@Import(TwitterStreamSupplierConfiguration.class)
|
||||
public static class TestTwitterStreamSourceApplication {
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionProperties properties,
|
||||
Function<TwitterConnectionProperties, ConfigurationBuilder> toConfigurationBuilder) {
|
||||
|
||||
Function<TwitterConnectionProperties, ConfigurationBuilder> mockedConfiguration =
|
||||
toConfigurationBuilder.andThen(
|
||||
new TwitterTestUtils().mockTwitterUrls(
|
||||
String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT)));
|
||||
|
||||
return mockedConfiguration.apply(properties).build();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.app.source.twitter.stream;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.function.Function;
|
||||
|
||||
import twitter4j.conf.ConfigurationBuilder;
|
||||
|
||||
import org.springframework.core.io.DefaultResourceLoader;
|
||||
import org.springframework.util.StreamUtils;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
public class TwitterTestUtils {
|
||||
|
||||
public Function<ConfigurationBuilder, ConfigurationBuilder> mockTwitterUrls(String baseUrl) {
|
||||
return configBuilder -> {
|
||||
configBuilder.setRestBaseURL(baseUrl + "/");
|
||||
configBuilder.setStreamBaseURL(baseUrl + "/stream/");
|
||||
configBuilder.setUserStreamBaseURL(baseUrl + "/user/");
|
||||
configBuilder.setSiteStreamBaseURL(baseUrl + "/site/");
|
||||
configBuilder.setUploadBaseURL(baseUrl + "/upload/");
|
||||
|
||||
configBuilder.setOAuthAccessTokenURL(baseUrl + "/oauth/access_token");
|
||||
configBuilder.setOAuthAuthenticationURL(baseUrl + "/oauth/authenticate");
|
||||
configBuilder.setOAuthAuthorizationURL(baseUrl + "/oauth/authorize");
|
||||
configBuilder.setOAuthRequestTokenURL(baseUrl + "/oauth/request_token");
|
||||
configBuilder.setOAuth2TokenURL(baseUrl + "/oauth2/token");
|
||||
configBuilder.setOAuth2InvalidateTokenURL(baseUrl + "/oauth2/invalidate_token");
|
||||
|
||||
return configBuilder;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Load Spring Resource as String.
|
||||
* @param resourcePath Resource path (accepts file:// , classpath:// and http:// uri schemas)
|
||||
* @return Returns text (UTF8) representation of the resource pointed by the resourcePath
|
||||
*/
|
||||
public static String asString(String resourcePath) {
|
||||
try {
|
||||
return StreamUtils.copyToString(new DefaultResourceLoader().getResource(resourcePath).getInputStream(),
|
||||
Charset.forName("UTF-8"));
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new RuntimeException("Can not load resource:" + resourcePath, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"created_at":"Thu Dec 20 13:56:10 +0000 2018","id":1075751718749659136,"id_str":"1075751718749659136","text":"Codementor: Kubernetes for Python Developers: Part 1\n#100daysofcode #python https:\/\/t.co\/VviTgpFpge","source":"\u003ca href=\"https:\/\/ifttt.com\" rel=\"nofollow\"\u003eIFTTT\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":859252650512072704,"id_str":"859252650512072704","name":"Freelancing|WebDev","screen_name":"FreelanceForBTC","location":"THE NET","url":"http:\/\/bit.ly\/BTCFREELANCING","description":"This twitter is designed to give the best information from web development to Blockchain.\n|YouTube's |Jobs |learning contented","translator_type":"none","protected":false,"verified":false,"followers_count":146,"friends_count":118,"listed_count":4,"favourites_count":445,"statuses_count":7528,"created_at":"Tue May 02 03:46:10 +0000 2017","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1071607798771793920\/UegrCs84_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1071607798771793920\/UegrCs84_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/859252650512072704\/1544326203","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"100daysofcode","indices":[53,67]},{"text":"python","indices":[68,75]}],"urls":[{"url":"https:\/\/t.co\/VviTgpFpge","expanded_url":"https:\/\/ift.tt\/2GxdMZF","display_url":"ift.tt\/2GxdMZF","indices":[76,99]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"ca","timestamp_ms":"1545314170907"}
|
||||
Reference in New Issue
Block a user