From c54bec0d4e029fe34926ef3258a86ccacc0d0182 Mon Sep 17 00:00:00 2001 From: twitter-team <> Date: Wed, 3 Sep 2025 15:46:53 -0500 Subject: [PATCH] update for-you recommendations code --- README.md | 48 +- .../scala/com/twitter/home_mixer/BUILD.bazel | 31 +- .../twitter/home_mixer/HomeMixerServer.scala | 79 +- .../HomeMixerThriftServerWarmupHandler.scala | 78 +- .../home_mixer/candidate_pipeline/BUILD.bazel | 12 +- ...sationServiceCandidatePipelineConfig.scala | 49 +- ...erviceCandidatePipelineConfigBuilder.scala | 40 +- ...ionServiceResponseFeatureTransformer.scala | 8 +- .../EditedTweetsCandidatePipelineConfig.scala | 23 +- ...erifiedPromptCandidatePipelineConfig.scala | 75 + .../twitter/home_mixer/controller/BUILD.bazel | 1 + .../controller/HomeHttpController.scala | 94 + .../controller/HomeThriftController.scala | 26 + .../twitter/home_mixer/federated/BUILD.bazel | 12 +- .../federated/HomeMixerColumn.scala | 112 +- .../decorator/BUILD.bazel | 19 +- .../EntryPointPivotModuleDecorator.scala | 62 + .../ForYouTweetCandidateDecorator.scala | 64 + ...onversationServiceCandidateDecorator.scala | 17 +- ...eywordTrendsModuleCandidateDecorator.scala | 68 + ...nnedTweetBroadcastCandidateDecorator.scala | 71 + .../StoriesModuleCandidateDecorator.scala | 70 + .../TuneFeedModuleCandidateDecorator.scala | 104 ++ ...weetCarouselModuleCandidateDecorator.scala | 199 ++ ...ideoCarouselModuleCandidateDecorator.scala | 200 ++ .../decorator/builder/BUILD.bazel | 21 +- .../HomeAdsClientEventDetailsBuilder.scala | 3 +- .../HomeClientEventDetailsBuilder.scala | 31 +- .../builder/HomeClientEventInfoBuilder.scala | 9 +- ...omeConversationModuleMetadataBuilder.scala | 3 +- .../builder/HomeTweetTypePredicates.scala | 357 +++- .../KeywordTrendMetaDescriptionBuilder.scala | 46 + .../builder/VerifiedPromptBuilder.scala | 71 + .../AuthorChildFeedbackActionBuilder.scala | 4 +- .../decorator/urt/builder/BUILD.bazel | 5 +- .../BlockUserChildFeedbackActionBuilder.scala | 26 +- .../builder/ChildFeedbackActionBuilder.scala | 270 +++ .../builder/DebugSocialContextBuilder.scala | 55 + .../DontLikeFeedbackActionBuilder.scala | 67 +- .../builder/EngagerSocialContextBuilder.scala | 30 +- .../decorator/urt/builder/FeedbackUtil.scala | 6 +- .../HomeFeedbackActionInfoBuilder.scala | 59 +- .../urt/builder/HomeTweetContextBuilder.scala | 92 + .../HomeTweetSocialContextBuilder.scala | 25 +- .../builder/LikedBySocialContextBuilder.scala | 9 +- .../MuteUserChildFeedbackActionBuilder.scala | 22 +- ...DetailsNegativeFeedbackActionBuilder.scala | 69 + .../builder/PostFeedbackActionBuilder.scala | 90 + .../PostFollowupFeedbackActionBuilder.scala | 87 + ...levancePromptCandidateUrtItemBuilder.scala | 98 + ...eportTweetChildFeedbackActionBuilder.scala | 13 +- .../RetweeterChildFeedbackActionBuilder.scala | 4 +- .../ServedTypeSocialContextBuilder.scala | 67 + .../TuneFeedFeedbackActionInfoBuilder.scala | 32 + .../AncestorFeatureHydrator.scala | 65 + .../AuthorFeatureHydrator.scala | 95 + ...AuthorLargeEmbeddingsFeatureHydrator.scala | 148 ++ .../feature_hydrator/BUILD.bazel | 119 +- .../BasketballContextFeatureHydrator.scala | 65 + .../BroadcastStateFeatureHydrator.scala | 70 + ...oryDiversityRescoringFeatureHydrator.scala | 112 ++ .../ClipEmbeddingFeatureHydrator.scala | 72 + ...ingMediaUnderstandingFeatureHydrator.scala | 140 ++ .../ClipImageClusterIdFeatureHydrator.scala | 95 + ...ependentBulkCandidateFeatureHydrator.scala | 77 + .../DiversityRescoringFeatureHydrator.scala | 119 ++ ...EarlybirdSearchResultFeatureHydrator.scala | 75 + .../FeedbackHistoryQueryFeatureHydrator.scala | 22 +- ...lowableUttTopicsQueryFeatureHydrator.scala | 53 + .../FrsSeedUsersQueryFeatureHydrator.scala | 66 + .../GeoduckAuthorLocationHydrator.scala | 200 ++ .../GizmoduckAuthorFeatureHydrator.scala | 209 +++ .../GizmoduckUserQueryFeatureHydrator.scala | 70 +- .../GraphTwoHopFeatureHydrator.scala | 101 + .../GrokAnnotationsFeatureHydrator.scala | 206 +++ ...rokGorkContentCreatorFeatureHydrator.scala | 46 + ...ranslatedPostIsCachedFeatureHydrator.scala | 76 + .../HeartbeatOptimizerParamsHydrator.scala | 84 + ...avyRankerWeightsQueryFeatureHydrator.scala | 77 + ...dImageClusterIdsQueryFeatureHydrator.scala | 112 ++ ...dMediaClusterIdsQueryFeatureHydrator.scala | 113 ++ ...ssionBloomFilterQueryFeatureHydrator.scala | 52 +- ...tiveFeedbackTimeQueryFeatureHydrator.scala | 75 + .../ListIdsQueryFeatureHydrator.scala | 61 + .../MediaClusterIdFeatureHydrator.scala | 123 ++ .../MediaCompletionRateFeatureHydrator.scala | 106 ++ .../MultiModalEmbeddingsFeatureHydrator.scala | 107 ++ .../NamesFeatureHydrator.scala | 3 +- ...NaviClientConfigQueryFeatureHydrator.scala | 72 + ...ideoClientConfigQueryFeatureHydrator.scala | 38 + .../OnPremRealGraphQueryFeatureHydrator.scala | 49 + ...OptimizerWeightsQueryFeatureHydrator.scala | 93 + ...AuthorLargeEmbeddingsFeatureHydrator.scala | 161 ++ ...lTweetLargeEmbeddingsFeatureHydrator.scala | 161 ++ ...PersistenceStoreQueryFeatureHydrator.scala | 59 +- .../PhoenixRescoringFeatureHydrator.scala | 48 + .../PostContextFeatureHydrator.scala | 101 + .../RateLimitQueryFeatureHydrator.scala | 54 + ...hInNetworkScoresQueryFeatureHydrator.scala | 8 +- .../RealGraphQueryFeatureHydrator.scala | 57 + ...RealGraphViewerAuthorFeatureHydrator.scala | 135 ++ ...aphViewerRelatedUsersFeatureHydrator.scala | 94 + ...eEntityRealGraphQueryFeatureHydrator.scala | 52 + ...eInteractionGraphEdgeFeatureHydrator.scala | 60 + ...nGraphUserVertexQueryFeatureHydrator.scala | 48 + .../RequestQueryFeatureHydrator.scala | 14 +- .../RequestTimeQueryFeatureHydrator.scala | 122 ++ ...SGSValidSocialContextFeatureHydrator.scala | 10 +- ...sEngagementSimilarityFeatureHydrator.scala | 82 + ...stersLogFavBasedTweetFeatureHydrator.scala | 97 + ...SparseEmbeddingsQueryFeatureHydrator.scala | 82 + .../SimClustersUserTweetScoresHydrator.scala | 97 + ...rBasedTopAuthorsQueryFeatureHydrator.scala | 111 ++ .../SlopAuthorFeatureHydrator.scala | 80 + .../SpaceStateFeatureHydrator.scala | 78 + .../TSPInferredTopicFeatureHydrator.scala | 174 ++ ...nsformerPostEmbeddingFeatureHydrator.scala | 155 ++ ...tEntityServiceContentFeatureHydrator.scala | 254 +++ .../TweetEntityServiceFeatureHydrator.scala | 376 ++++ .../TweetLanguageFeatureHydrator.scala | 61 + .../TweetLargeEmbeddingsFeatureHydrator.scala | 145 ++ .../TweetMetaDataFeatureHydrator.scala | 61 + .../TweetTimeFeatureHydrator.scala | 129 ++ .../TweetTypeMetricsFeatureHydrator.scala | 50 + .../TweetypieFeatureHydrator.scala | 141 +- .../TwhinAuthorFollowFeatureHydrator.scala | 86 + .../TwhinRebuildTweetFeatureHydrator.scala | 127 ++ ...ldUserEngagementQueryFeatureHydrator.scala | 74 + ...uildUserPositiveQueryFeatureHydrator.scala | 67 + .../TwhinTweetFeatureHydrator.scala | 111 ++ ...inUserEngagementQueryFeatureHydrator.scala | 72 + .../TwhinUserFollowQueryFeatureHydrator.scala | 72 + .../TwhinUserNegativeFeatureHydrator.scala | 47 + .../TwhinUserPositiveFeatureHydrator.scala | 47 + .../TwhinVideoFeatureHydrator.scala | 54 + ...ActionsUserIdentifierFeatureHydrator.scala | 60 + ...rActionByteArrayQueryFeatureHydrator.scala | 99 + .../UserActionsQueryFeatureHydrator.scala | 156 ++ ...EngagedGrokCategoriesFeatureHydrator.scala | 45 + .../UserEngagedLanguagesFeatureHydrator.scala | 34 + ...UserEngagementGrokTagFeatureHydrator.scala | 57 + .../UserFrequentLocationHydrator.scala | 145 ++ ...FrequentLocationQueryFeatureHydrator.scala | 90 + ...sformerEmbeddingQueryFeatureHydrator.scala | 183 ++ .../UserLanguagesFeatureHydrator.scala | 46 + .../UserLargeEmbeddingsFeatureHydrator.scala | 142 ++ .../UserStateQueryFeatureHydrator.scala | 52 + ...UserSubscriptionQueryFeatureHydrator.scala | 37 + ...derstandableLangaugesFeatureHydrator.scala | 57 + .../UtegFeatureHydrator.scala | 102 + ...VideoSummaryEmbeddingFeatureHydrator.scala | 83 + .../ViewCountsFeatureHydrator.scala | 69 + ...ContentCreatorMetricsFeatureHydrator.scala | 42 + .../WithDefaultFeatureMap.scala | 8 + .../AuthorFeaturesAdapter.scala | 92 + .../adapters/author_features/BUILD.bazel | 17 + .../adapters/content/BUILD.bazel | 15 + .../ClipEmbeddingFeaturesAdapter.scala | 31 + .../content/ContentFeatureAdapter.scala | 284 +++ .../content/TextTokensFeaturesAdapter.scala | 31 + ...VideoSummaryEmbeddingFeaturesAdaptor.scala | 32 + .../adapters/gizmoduck_features/BUILD.bazel | 11 + .../GizmoduckFeaturesAdapter.scala | 50 + .../adapters/inferred_topic/BUILD.bazel | 11 + .../inferred_topic/InferredTopicAdapter.scala | 25 + .../light_ranking_features/BUILD.bazel | 12 + ...LightRankingCandidateFeaturesAdapter.scala | 64 + .../adapters/non_ml_features/BUILD.bazel | 13 + .../NonMLCandidateFeaturesAdapter.scala | 45 + .../NonMLCommonFeaturesAdapter.scala | 71 + .../adapters/offline_aggregates/BUILD.bazel | 12 + .../PassThroughAdapter.scala | 12 + .../SparseAggregatesToDenseAdapter.scala | 17 + .../adapters/simclusters_features/BUILD.bazel | 12 + .../SimclustersFeaturesAdapter.scala | 42 + .../transformer_embeddings/BUILD.bazel | 13 + .../TransformerEmbeddingsAdapter.scala | 129 ++ .../UserHistoryEventsAdapter.scala | 109 ++ .../VideoUserHistoryEventsAdapter.scala | 146 ++ .../adapters/twhin_embeddings/BUILD.bazel | 13 + .../TwhinEmbeddingsAdapter.scala | 152 ++ .../AggregateFeatureInfo.scala | 38 + .../offline_aggregates/BUILD.bazel | 24 + .../BaseEdgeAggregateFeatureHydrator.scala | 124 ++ .../EdgeAggregateFeatures.scala | 127 ++ .../PartAAggregateQueryFeatureHydrator.scala | 37 + .../PartBAggregateQueryFeatureHydrator.scala | 146 ++ .../TopicEdgeAggregateFeatureHydrator.scala | 26 + ...dgeTruncatedAggregateFeatureHydrator.scala | 24 + ...tContentEdgeAggregateFeatureHydrator.scala | 14 + ...rEngagerEdgeAggregateFeatureHydrator.scala | 16 + ...erEntityEdgeAggregateFeatureHydrator.scala | 18 + .../offline_aggregates/Utils.scala | 36 + .../real_time_aggregates/BUILD.bazel | 35 + ...thorRealTimeAggregateFeatureHydrator.scala | 60 + ...ggregateBulkCandidateFeatureHydrator.scala | 118 ++ ...mentRealTimeAggregateFeatureHydrator.scala | 79 + ...mentRealTimeAggregateFeatureHydrator.scala | 74 + ...mentRealTimeAggregateFeatureHydrator.scala | 144 ++ ...mentRealTimeAggregateFeatureHydrator.scala | 147 ++ ...mentRealTimeAggregateFeatureHydrator.scala | 59 + ...mentRealTimeAggregateFeatureHydrator.scala | 147 ++ ...entRealTimeAggregatesFeatureHydrator.scala | 159 ++ ...ideoRealTimeAggregateFeatureHydrator.scala | 96 + .../feature_hydrator/user_history/BUILD.bazel | 32 + ...serHistoryEventsQueryFeatureHydrator.scala | 57 + ...serHistoryEventsQueryFeatureHydrator.scala | 39 + ...serHistoryEventsQueryFeatureHydrator.scala | 339 ++++ .../filter/AuthorDedupFilter.scala | 34 + .../functional_component/filter/BUILD.bazel | 16 +- .../ClipClusterDeduplicationFilter.scala | 59 + .../filter/ClusterBasedDedupFilter.scala | 138 ++ .../filter/ConsistentAspectRatioFilter.scala | 57 + .../filter/CountryFilter.scala | 67 + .../filter/CurrentPinnedTweetFilter.scala | 31 + .../filter/FeedbackFatigueFilter.scala | 8 +- .../filter/GrokGoreFilter.scala | 39 + .../filter/GrokNsfwFilter.scala | 47 + .../filter/GrokSpamFilter.scala | 39 + .../filter/GrokViolentFilter.scala | 39 + .../filter/HasAuthorFilter.scala | 25 + .../filter/HasMultipleMediaFilter.scala | 35 + .../InvalidSubscriptionTweetFilter.scala | 27 +- .../filter/LocationFilter.scala | 34 + .../filter/MaxVideoDurationFilter.scala | 44 + .../filter/MediaDeduplicationFilter.scala | 42 + .../filter/MinVideoDurationFilter.scala | 61 + .../filter/PreviouslySeenMediaIdsFilter.scala | 38 + .../filter/PreviouslySeenTweetsFilter.scala | 30 +- .../filter/PreviouslyServedTweetsFilter.scala | 5 +- .../filter/QuoteDeduplicationFilter.scala | 30 + .../filter/RegionFilter.scala | 57 + .../filter/ReplyFilter.scala | 15 +- .../filter/RetweetDeduplicationFilter.scala | 4 +- .../filter/SlopFilter.scala | 68 + .../filter/TweetHydrationFilter.scala | 29 + .../filter/WeeklyBookmarkFilter.scala | 34 + .../gate/AllowForYouRecommendationsGate.scala | 24 + .../functional_component/gate/BUILD.bazel | 10 +- .../gate/BookmarksTimeGate.scala | 38 + .../gate/ExcludeSyntheticUserGate.scala | 22 + ...rsistenceStoreDurationValidationGate.scala | 53 + .../gate/RateLimitGate.scala | 19 + .../gate/RateLimitNotGate.scala | 19 + .../gate/RecentlyServedByServedTypeGate.scala | 58 + .../gate/TestUserProbabilisticGate.scala | 33 + ...nesPersistenceStoreLastInjectionGate.scala | 6 +- .../functional_component/scorer/BUILD.bazel | 14 +- .../scorer/FeedbackFatigueScorer.scala | 8 +- .../scorer/NaviModelScorer.scala | 142 ++ .../scorer/PhoenixModelRerankingScorer.scala | 81 + .../scorer/PhoenixScorer.scala | 85 + .../scorer/PredictClientFactory.scala | 83 + .../scorer/WeighedModelRerankingScorer.scala | 71 + .../functional_component/selector/BUILD.bazel | 9 +- .../selector/RandomShuffleCandidates.scala | 23 + .../ScoreAveragingPositionSelector.scala | 238 +++ .../SortFixedPositionCandidates.scala | 57 + .../UpdateHomeClientEventDetails.scala | 34 +- .../side_effect/BUILD.bazel | 45 +- ...BaseCacheCandidateFeaturesSideEffect.scala | 468 +++++ .../side_effect/ClientEventsBuilder.scala | 57 +- .../CommonFeaturesPldrConverter.scala | 140 ++ .../HomeScribeClientEventSideEffect.scala | 21 +- ...HomeScribeServedCandidatesSideEffect.scala | 149 +- ...entSentImpressionsEventBusSideEffect.scala | 18 +- ...eTimelinesPersistenceStoreSideEffect.scala | 290 +-- .../HomeMixerDebugParamsUnmarshaller.scala | 3 +- .../HomeMixerProductContextUnmarshaller.scala | 39 +- .../HomeMixerProductUnmarshaller.scala | 12 +- .../marshaller/timeline_logging/BUILD.bazel | 2 +- .../TweetDetailsMarshaller.scala | 4 +- .../com/twitter/home_mixer/model/BUILD.bazel | 20 +- .../model/ClearCacheIncludeInstruction.scala | 27 +- .../home_mixer/model/ContentFeatures.scala | 8 +- .../twitter/home_mixer/model/GrokTopics.scala | 38 + .../home_mixer/model/HomeFeatures.scala | 370 +++- .../model/HomeLargeEmbeddingsFeatures.scala | 50 + .../home_mixer/model/NaviClientConfig.scala | 6 + .../model/NavigationIncludeInstruction.scala | 41 + .../model/PhoenixPredictedScoreFeature.scala | 193 ++ .../model/PredictedScoreFeature.scala | 336 ++++ .../model/candidate_source/BUILD.bazel | 8 + .../model/candidate_source/SourceSignal.scala | 9 + .../home_mixer/model/request/BUILD.bazel | 4 +- .../model/request/HomeMixerDebugOptions.scala | 3 +- .../model/request/HomeMixerProduct.scala | 12 +- .../request/HomeMixerProductContext.scala | 33 +- .../home_mixer/model/signup/BUILD.bazel | 8 + .../model/signup/SignupSource.scala | 6 + .../com/twitter/home_mixer/module/BUILD.bazel | 68 +- .../module/ClusterDetailsModule.scala | 45 + .../module/EarlybirdRealtimeCGModule.scala | 48 + .../module/EventsRecosClientModule.scala | 30 + .../GizmoduckTimelinesCacheClientModule.scala | 70 + .../module/HomeMixerFeaturesModule.scala | 82 + .../module/HomeMixerFlagsModule.scala | 10 +- .../module/HomeMixerResourcesModule.scala | 29 +- .../module/InMemoryCacheModule.scala | 64 + .../home_mixer/module/LimiterModule.scala | 27 + .../module/ManhattanClientsModule.scala | 37 + .../ManhattanFeatureRepositoryModule.scala | 78 +- .../module/MediaClusterId88Module.scala | 21 + .../module/MediaClusterIdModule.scala | 21 + .../MemcachedFeatureRepositoryModule.scala | 53 +- ...edScoredCandidateFeaturesStoreModule.scala | 84 + .../module/NaviModelClientModule.scala | 65 +- .../module/OptimizedStratoClientModule.scala | 101 +- .../module/PhoenixClientModule.scala | 61 + .../RealGraphInNetworkScoresModule.scala | 5 +- ...timeAggregateFeatureRepositoryModule.scala | 52 +- .../module/ScoredTweetsMemcacheModule.scala | 7 +- .../ScoredVideoTweetsMemcacheModule.scala | 58 + .../module/ScribeEventPublisherModule.scala | 91 +- .../ThriftFeatureRepositoryModule.scala | 93 +- .../TvWatchHistoryCacheClientModule.scala | 72 + .../module/TweetWatchTimeMetadataModule.scala | 23 + ...typieStaticEntitiesCacheClientModule.scala | 80 +- .../module/TwhinEmbeddingsModule.scala | 62 + .../home_mixer/module/UttTopicModule.scala | 106 ++ .../module/VideoEmbeddingModule.scala | 22 + .../com/twitter/home_mixer/param/BUILD.bazel | 1 + .../param/HomeGlobalParamConfig.scala | 262 ++- .../home_mixer/param/HomeGlobalParams.scala | 1421 +++++++++++++- .../home_mixer/param/HomeMixerFlagName.scala | 4 +- .../param/HomeMixerInjectionNames.scala | 47 +- .../home_mixer/param/decider/BUILD.bazel | 4 +- .../home_mixer/param/decider/DeciderKey.scala | 55 +- .../twitter/home_mixer/product/BUILD.bazel | 13 +- .../HomeProductPipelineRegistryConfig.scala | 26 +- .../home_mixer/product/following/BUILD.bazel | 51 +- ...FollowingAdsCandidatePipelineBuilder.scala | 44 +- ...AdsDependentCandidatePipelineBuilder.scala | 111 ++ ...owingDependentAdsMixerPipelineConfig.scala | 315 ++++ ...wingEarlybirdCandidatePipelineConfig.scala | 8 +- .../FollowingEarlybirdQueryTransformer.scala | 3 - .../FollowingMixerPipelineConfig.scala | 137 +- .../FollowingProductPipelineConfig.scala | 17 +- ...FollowCandidatePipelineConfigBuilder.scala | 6 +- .../product/following/model/BUILD.bazel | 5 - .../model/HomeMixerExternalStrings.scala | 100 +- .../product/following/param/BUILD.bazel | 5 +- .../following/param/FollowingParam.scala | 119 +- .../param/FollowingParamConfig.scala | 33 +- .../home_mixer/product/for_you/BUILD.bazel | 75 +- .../ForYouAdsCandidatePipelineBuilder.scala | 25 +- ...AdsDependentCandidatePipelineBuilder.scala | 14 +- ...rYouBookmarksCandidatePipelineConfig.scala | 88 + ...unitiesToJoinCandidatePipelineConfig.scala | 130 ++ ...ryPointPivotCandidatePipelineBuilder.scala | 91 + ...tryPointPivotCandidatePipelineConfig.scala | 81 + ...orationTweetsCandidatePipelineConfig.scala | 177 ++ ...uelFrameFrameCandidatePipelineConfig.scala | 89 + ...KeywordTrendsCandidatePipelineConfig.scala | 83 + .../for_you/ForYouMixerPipelineConfig.scala | 584 ++++++ ...uPinnedTweetsCandidatePipelineConfig.scala | 114 ++ .../for_you/ForYouProductPipelineConfig.scala | 33 +- ...commendedJobsCandidatePipelineConfig.scala | 97 + ...OrganizationsCandidatePipelineConfig.scala | 102 + ...levancePromptCandidatePipelineConfig.scala | 66 + .../ForYouResponseDomainMarshaller.scala | 84 + ...uScoredTweetsCandidatePipelineConfig.scala | 118 +- ...oredTweetsResponseFeatureTransformer.scala | 206 ++- ...edVideoTweetsCandidatePipelineConfig.scala | 102 + ...ForYouStoriesCandidatePipelineConfig.scala | 69 + ...orYouTuneFeedCandidatePipelineConfig.scala | 101 + ...TweetPreviewsCandidatePipelineConfig.scala | 40 +- ...FollowCandidatePipelineConfigBuilder.scala | 15 +- ...scribeCandidatePipelineConfigBuilder.scala | 4 +- .../for_you/candidate_source/BUILD.bazel | 21 +- .../BookmarksCandidateSource.scala | 46 + ...oadcastedPinnedTweetsCandidateSource.scala | 66 + .../JetfuelFrameCandidateSource.scala | 37 + .../RecommendedJobsCandidateSource.scala | 37 + ...cruitingOrganizationsCandidateSource.scala | 37 + .../ScoredTweetsProductCandidateSource.scala | 99 +- ...etsCategorizedProductCandidateSource.scala | 73 + ...redVideoTweetsProductCandidateSource.scala | 84 + .../StoriesModuleCandidateSource.scala | 53 + .../TuneFeedCandidateSource.scala | 48 + .../UnifiedTrendsCandidateSource.scala | 34 + .../ArticlePreviewTextFeatureHydrator.scala | 52 + .../for_you/feature_hydrator/BUILD.bazel | 23 +- .../CurrentPinnedTweetFeatureHydrator.scala | 54 + ...splayedGrokTopicQueryFeatureHydrator.scala | 37 + ...ingSportsAccountQueryFeatureHydrator.scala | 51 + ...ineServiceTweetsQueryFeatureHydrator.scala | 42 +- .../TweetAuthorFeatureHydrator.scala | 93 + .../TweetAuthorFollowersFeatureHydrator.scala | 69 + .../TweetEngagementsFeatureHydrator.scala | 91 + ...iewTweetypieCandidateFeatureHydrator.scala | 103 +- ...HasJobRecommendationsFeatureHydrator.scala | 82 + .../product/for_you/filter/BUILD.bazel | 6 +- .../for_you/filter/NotArticleFilter.scala | 29 + .../for_you/filter/PromotedTrendFilter.scala | 31 + .../filter/TweetPreviewTextFilter.scala | 5 +- .../product/for_you/gate/BUILD.bazel | 18 + .../gate/FollowingSportsUsersGate.scala | 20 + .../for_you/gate/TuneFeedModuleGate.scala | 21 + .../for_you/gate/UserFollowingRangeGate.scala | 32 + .../product/for_you/model/BUILD.bazel | 4 - .../product/for_you/model/ForYouQuery.scala | 3 +- .../product/for_you/param/BUILD.bazel | 6 +- .../product/for_you/param/ForYouParam.scala | 641 ++++++- .../for_you/param/ForYouParamConfig.scala | 114 +- .../for_you/query_transformer/BUILD.bazel | 3 +- .../UnifiedCandidatesQueryTransformer.scala | 34 + .../for_you/response_transformer/BUILD.bazel | 8 +- .../BookmarksResponseFeatureTransformer.scala | 25 + ...ationTweetResponseFeatureTransformer.scala | 26 + .../KeywordTrendsFeatureTransformer.scala | 53 + ...innedTweetResponseFeatureTransformer.scala | 30 + ...VideoTweetResponseFeatureTransformer.scala | 38 + ...riesModuleResponseFeatureTransformer.scala | 51 + .../TuneFeedFeatureTransformer.scala | 20 + ...eetPreviewResponseFeatureTransformer.scala | 8 +- .../product/for_you/scorer/BUILD.bazel | 2 - .../scorer/PinnedTweetCandidateScorer.scala | 31 + .../product/for_you/selector/BUILD.bazel | 11 + .../selector/DebugUpdateSortAdsResult.scala | 31 + .../for_you/selector/DebunchCandidates.scala | 70 + ...moveDuplicateCandidatesOutsideModule.scala | 46 + .../product/for_you/side_effect/BUILD.bazel | 6 +- ...dCandidateFeatureKeysKafkaSideEffect.scala | 77 +- .../side_effect/ServedStatsSideEffect.scala | 143 +- .../VideoServedStatsSideEffect.scala | 124 ++ .../product/scored_tweets/BUILD.bazel | 21 +- .../ScoredTweetsProductPipelineConfig.scala | 16 +- ...edTweetsRecommendationPipelineConfig.scala | 545 ++++-- .../candidate_pipeline/BUILD.bazel | 7 +- ...dScoredTweetsCandidatePipelineConfig.scala | 9 + ...weetsBackfillCandidatePipelineConfig.scala | 30 +- ...ntExplorationCandidatePipelineConfig.scala | 75 + ...etsDirectUtegCandidatePipelineConfig.scala | 89 + ...edTweetsListsCandidatePipelineConfig.scala | 35 +- ...dTweetsStaticCandidatePipelineConfig.scala | 77 + ...etsTweetMixerCandidatePipelineConfig.scala | 63 +- .../candidate_pipeline/earlybird/BUILD.bazel | 10 +- ...tsCommunitiesCandidatePipelineConfig.scala | 76 + ...birdInNetworkCandidatePipelineConfig.scala | 45 +- .../candidate_source/BUILD.bazel | 11 +- .../ContentExplorationCandidateSource.scala | 79 + ...lybirdRealtimeCGTweetCandidateSource.scala | 27 + .../ListsCandidateSource.scala | 20 +- .../StaticPostsCandidateSource.scala | 57 + .../feature_hydrator/BUILD.bazel | 54 +- ...chedScoredTweetsQueryFeatureHydrator.scala | 18 +- .../EarlybirdFeatureHydrator.scala | 205 +- .../FollowedUserScoresFeatureHydrator.scala | 65 + ...chedScoredTweetsQueryFeatureHydrator.scala | 46 + .../IsColdStartPostFeatureHydrator.scala | 82 + .../ListNameFeatureHydrator.scala | 48 + .../LowSignalUserQueryFeatureHydrator.scala | 90 + .../ReplyFeatureHydrator.scala | 104 +- .../SGSMutuallyFollowedUserHydrator.scala | 58 + .../SemanticCoreFeatureHydrator.scala | 112 ++ .../TweetypieVisibilityFeatureHydrator.scala | 255 +++ .../ValidLikedByUserIdsFeatureHydrator.scala | 34 + .../adapters/content/BUILD.bazel | 5 - .../adapters/earlybird/BUILD.bazel | 4 - .../adapters/earlybird/EarlybirdAdapter.scala | 8 +- .../product/scored_tweets/filter/BUILD.bazel | 14 +- .../filter/ControlAiExcludeFilter.scala | 42 + .../filter/ControlAiOnlyIncludeFilter.scala | 42 + .../filter/CustomSnowflakeIdAgeFilter.scala | 49 + ...mSnowflakeIdAgeFilterWithIncludeRule.scala | 56 + ...stomSnowflakeIdAgeFilterWithSkipRule.scala | 54 + .../DuplicateConversationTweetsFilter.scala | 16 +- .../filter/ExtendedDirectedAtFilter.scala | 36 + .../filter/FollowedAuthorFilter.scala | 31 + .../GrokAutoTranslateLanguageFilter.scala | 64 + .../IsOutOfNetworkColdStartPostFilter.scala | 34 + .../scored_tweets/filter/LanguageFilter.scala | 69 + .../scored_tweets/filter/OONReplyFilter.scala | 43 + .../filter/QualifiedRepliesFilter.scala | 45 + .../filter/SGSAuthorFilter.scala | 60 + .../scored_tweets/filter/TopKFilter.scala | 31 + .../filter/TopKOptionalFilter.scala | 32 + .../filter/UtegMinFavCountFilter.scala | 32 + .../scored_tweets/filter/UtegTopKFilter.scala | 35 + .../gate/AllowLowSignalUserGate.scala | 26 + .../product/scored_tweets/gate/BUILD.bazel | 1 + .../gate/DenyLowSignalUserGate.scala | 30 + .../gate/MatchesCountryGate.scala | 30 + .../gate/MinCachedTweetsGate.scala | 11 +- .../gate/RecentFeedbackCheckGate.scala | 27 + .../scored_tweets/marshaller/BUILD.bazel | 6 +- ...ScoredTweetsResponseDomainMarshaller.scala | 13 +- ...redTweetsResponseTransportMarshaller.scala | 106 +- .../product/scored_tweets/model/BUILD.bazel | 3 - .../model/ScoredTweetsQuery.scala | 10 +- .../model/ScoredTweetsResponse.scala | 13 +- .../product/scored_tweets/param/BUILD.bazel | 1 - .../param/ScoredTweetsParam.scala | 944 +++++++--- .../param/ScoredTweetsParamConfig.scala | 162 +- .../query_transformer/BUILD.bazel | 5 +- .../ContentExplorationQueryTransformer.scala | 26 + .../TimelineRankerFrsQueryTransformer.scala | 10 +- ...elineRankerInNetworkQueryTransformer.scala | 8 +- .../TimelineRankerQueryTransformer.scala | 15 +- .../TimelineRankerUtegQueryTransformer.scala | 8 +- .../UtegQueryTransformer.scala | 68 + .../query_transformer/earlybird/BUILD.bazel | 7 +- ...CommunitiesEarlybirdQueryTransformer.scala | 124 ++ .../EarlybirdFrsQueryTransformer.scala | 32 +- .../EarlybirdInNetworkQueryTransformer.scala | 46 +- .../earlybird/EarlybirdQueryTransformer.scala | 65 +- .../response_transformer/BUILD.bazel | 6 +- ...oredTweetsResponseFeatureTransformer.scala | 175 +- ...tsBackfillResponseFeatureTransformer.scala | 15 +- ...xplorationResponseFeatureTransformer.scala | 35 + ...DirectUtegResponseFeatureTransformer.scala | 44 + ...dTweetsFrsResponseFeatureTransformer.scala | 9 +- ...sInNetworkResponseFeatureTransformer.scala | 24 +- ...weetsListsResponseFeatureTransformer.scala | 35 +- ...eVideoRecoResponseFeatureTransformer.scala | 27 + ...eetsStaticResponseFeatureTransformer.scala | 33 + ...TweetMixerResponseFeatureTransformer.scala | 200 +- ...innedTweetResponseFeatureTransformer.scala | 26 + .../TimelineRankerResponseTransformer.scala | 16 +- .../earlybird/BUILD.bazel | 3 +- .../EarlybirdResponseTransformer.scala | 71 +- ...ommunitiesResponseFeatureTransformer.scala | 42 + ...rlybirdFrsResponseFeatureTransformer.scala | 23 +- ...dInNetworkResponseFeatureTransformer.scala | 24 +- ...lybirdNsfwResponseFeatureTransformer.scala | 40 + ...AuthorBasedListwiseRescoringProvider.scala | 55 + .../product/scored_tweets/scorer/BUILD.bazel | 17 +- ...ceDiversityListwiseRescoringProvider.scala | 50 + ...ExplorationListwiseRescoringProvider.scala | 49 + .../scorer/ControlAiRescorer.scala | 55 + ...epRetrievalListwiseRescoringProvider.scala | 48 + ...CrossBorderListwiseRescoringProvider.scala | 48 + ...epRetrievalListwiseRescoringProvider.scala | 48 + .../scorer/GrokSlopScoreRescorer.scala | 43 + .../scorer/HeuristicScorer.scala | 60 +- ...mpressedAuthorDecayRescoringProvider.scala | 85 + ...lusterBasedListwiseRescoringProvider.scala | 80 + ...lusterBasedListwiseRescoringProvider.scala | 79 + .../scorer/ListwiseRescoringProvider.scala | 54 + .../scorer/LowSignalScorer.scala | 60 + .../scorer/MultimodalEmbeddingRescorer.scala | 88 + .../scorer/RescoringFactorProvider.scala | 121 +- .../scoring_pipeline/BUILD.bazel | 16 +- ...TweetsHeuristicScoringPipelineConfig.scala | 18 +- ...TweetsLowSignalScoringPipelineConfig.scala | 26 + ...oredTweetsModelScoringPipelineConfig.scala | 272 ++- ...TweetsRerankingScoringPipelineConfig.scala | 40 + .../scored_tweets/selector/BUILD.bazel | 5 +- .../KeepTopKCandidatesPerCommunity.scala | 37 + .../scored_tweets/side_effect/BUILD.bazel | 26 +- .../CacheCandidateFeaturesSideEffect.scala | 45 + .../CacheRequestInfoSideEffect.scala | 73 + .../CacheRetrievalSignalSideEffect.scala | 116 ++ .../CachedScoredTweetsSideEffect.scala | 135 +- .../CommonFeaturesSideEffect.scala | 96 + ...dCandidateFeatureKeysKafkaSideEffect.scala | 257 +++ ...CandidateScoreFeatureKafkaSideEffect.scala | 122 ++ ...oredPhoenixCandidatesKafkaSideEffect.scala | 188 ++ .../side_effect/ScoredStatsSideEffect.scala | 562 ++++++ ...ScoredTweetsDiversityStatsSideEffect.scala | 81 + .../ScribeScoredCandidatesSideEffect.scala | 63 +- .../product/scored_tweets/util/BUILD.bazel | 12 + .../scored_tweets/util/ControlAiUtil.scala | 101 + .../home_mixer/product/subscribed/BUILD.bazel | 48 +- ...ibedEarlybirdCandidatePipelineConfig.scala | 6 +- .../SubscribedMixerPipelineConfig.scala | 53 +- .../SubscribedProductPipelineConfig.scala | 2 +- .../product/subscribed/model/BUILD.bazel | 6 - .../product/subscribed/param/BUILD.bazel | 4 - .../subscribed/param/SubscribedParam.scala | 7 + .../param/SubscribedParamConfig.scala | 4 + .../twitter/home_mixer/service/BUILD.bazel | 5 - .../service/HomeMixerAccessPolicy.scala | 2 +- .../service/HomeMixerAlertConfig.scala | 3 +- .../service/ScoredTweetsService.scala | 1 + .../com/twitter/home_mixer/store/BUILD.bazel | 19 +- .../store/MediaClusterId88Store.scala | 19 + .../store/MediaClusterId95Store.scala | 19 + .../store/MediaClusterIdStoreTrait.scala | 102 + .../twitter/home_mixer/store/RTAMHStore.scala | 39 + .../store/RealGraphInNetworkScoresStore.scala | 2 +- .../store/TweetWatchTimeMetadataStore.scala | 122 ++ .../store/TwhinEmbeddingsStore.scala | 195 ++ .../store/VideoEmbeddingMHStore.scala | 106 ++ .../com/twitter/home_mixer/util/BUILD.bazel | 15 +- .../util/CachedScoredTweetsHelper.scala | 5 +- .../home_mixer/util/CandidatesUtil.scala | 45 +- .../home_mixer/util/LanguageCode.scala | 92 + .../util/NaviScorerStatsHandler.scala | 107 ++ .../util/ObservedKeyValueResultHandler.scala | 4 +- .../util/PhoenixScorerStatsHandler.scala | 88 + .../home_mixer/util/PhoenixUtils.scala | 161 ++ .../home_mixer/util/RerankerUtil.scala | 138 ++ .../twitter/home_mixer/util/SignalUtil.scala | 26 + .../home_mixer/util/TensorFlowUtil.scala | 6 + .../com/twitter/home_mixer/util/UrtUtil.scala | 33 + .../home_mixer/util/earlybird/BUILD.bazel | 6 +- .../util/earlybird/EarlybirdRequestUtil.scala | 47 +- .../earlybird/EarlybirdResponseUtil.scala | 92 +- .../util/earlybird/RelevanceSearchUtil.scala | 8 +- .../util/tweetypie/content/BUILD.bazel | 10 - .../content/FeatureExtractionHelper.scala | 6 +- .../content/TweetMediaFeaturesExtractor.scala | 158 +- .../scala/com/twitter/tweet_mixer/BUILD.bazel | 26 + .../TweetMixerHttpServerWarmupHandler.scala | 20 + .../tweet_mixer/TweetMixerServer.scala | 169 ++ .../TweetMixerThriftServerWarmupHandler.scala | 77 + .../candidate_pipeline/BUILD.bazel | 61 + ...TweetsCandidatePipelineConfigFactory.scala | 93 + ...tBasedCandidatePipelineConfigFactory.scala | 139 ++ ...tTweetCandidatePipelineConfigFactory.scala | 196 ++ ...ierTwoCandidatePipelineConfigFactory.scala | 195 ++ ...onDRUserTweetCandidatePipelineConfig.scala | 95 + ...rTweetTierTwoCandidatePipelineConfig.scala | 95 + ...larityCandidatePipelineConfigFactory.scala | 208 +++ ...ierTwoCandidatePipelineConfigFactory.scala | 210 +++ ...tTweetCandidatePipelineConfigFactory.scala | 185 ++ ...imclusterColdCandidatePipelineConfig.scala | 74 + ...ontrolAiTopicCandidatePipelineConfig.scala | 94 + ...ngaugeCandidatePipelineConfigFactory.scala | 75 + ...larityCandidatePipelineConfigFactory.scala | 153 ++ ...larityCandidatePipelineConfigFactory.scala | 107 ++ ...eetSimilarityCandidatePipelineConfig.scala | 131 ++ ...etworkCandidatePipelineConfigFactory.scala | 86 + .../EventsCandidatePipelineConfig.scala | 68 + ...rderUserTweetCandidatePipelineConfig.scala | 94 + ...enDRUserTweetCandidatePipelineConfig.scala | 94 + ...VideosCandidatePipelineConfigFactory.scala | 88 + ...loliteCandidatePipelineConfigFactory.scala | 90 + ...larityCandidatePipelineConfigFactory.scala | 130 ++ ...larityCandidatePipelineConfigFactory.scala | 191 ++ ...tRelatedCreatorPipelineConfigFactory.scala | 89 + ...TweetsCandidatePipelineConfigFactory.scala | 73 + ...TweetsCandidatePipelineConfigFactory.scala | 81 + ...TweetsCandidatePipelineConfigFactory.scala | 84 + ...TweetsCandidatePipelineConfigFactory.scala | 108 ++ ...cVideoCandidatePipelineConfigFactory.scala | 91 + ...stedInCandidatePipelineConfigFactory.scala | 108 ++ ...rBasedCandidatePipelineConfigFactory.scala | 103 ++ ...reatorCandidatePipelineConfigFactory.scala | 105 ++ ...tBasedCandidatePipelineConfigFactory.scala | 102 + ...TweetsCandidatePipelineConfigFactory.scala | 86 + .../TrendsVideoCandidatePipelineConfig.scala | 69 + ...larityCandidatePipelineConfigFactory.scala | 100 + ...larityCandidatePipelineConfigFactory.scala | 93 + ...rBasedCandidatePipelineConfigFactory.scala | 105 ++ ...eetSimilarityCandidatePipelineConfig.scala | 94 + ...gVideoCandidatePipelineConfigFactory.scala | 89 + ...tVideoCandidatePipelineConfigFactory.scala | 91 + .../UTEGCandidatePipelineConfigFactory.scala | 85 + ...tBasedCandidatePipelineConfigFactory.scala | 125 ++ ...rBasedCandidatePipelineConfigFactory.scala | 94 + ...tBasedCandidatePipelineConfigFactory.scala | 125 ++ ...tBasedCandidatePipelineConfigFactory.scala | 123 ++ ...tBasedCandidatePipelineConfigFactory.scala | 126 ++ ...ummaryCandidatePipelineConfigFactory.scala | 151 ++ .../UserLocationCandidatePipelineConfig.scala | 79 + .../tweet_mixer/candidate_source/BUILD.bazel | 8 + .../candidate_source/UTG/BUILD.bazel | 19 + .../UTG/UTGProducerBasedRequest.scala | 15 + .../UTG/UTGTweetBasedRequest.scala | 16 + ...eetGraphConsumerBasedCandidateSource.scala | 114 ++ ...eetGraphProducerBasedCandidateSource.scala | 97 + ...rTweetGraphTweetBasedCandidateSource.scala | 98 + .../candidate_source/UVG/BUILD.bazel | 19 + .../UVG/UVGTweetBasedRequest.scala | 20 + ...deoGraphConsumerBasedCandidateSource.scala | 115 ++ ...rVideoGraphTweetBasedCandidateSource.scala | 106 ++ .../cached_candidate_source/BUILD.bazel | 12 + .../MemcachedCandidateSource.scala | 90 + .../content_embedding_ann/BUILD.bazel | 16 + .../ContentEmbeddingAnnCandidateSource.scala | 88 + .../curated_user_tls_per_language/BUILD.bazel | 12 + ...tedUserTlsPerLanguageCandidateSource.scala | 31 + .../earlybird_realtime_cg/BUILD.bazel | 15 + ...lybirdRealtimeCGTweetCandidateSource.scala | 104 ++ .../InNetworkRequest.scala | 8 + .../engaged_users/BUILD.bazel | 12 + .../RecentEngagedUsersCandidateSource.scala | 35 + .../candidate_source/events/BUILD.bazel | 15 + .../events/EventsCandidateSource.scala | 67 + .../evergreen_videos/BUILD.bazel | 19 + .../EvergreenVideosSearchByTweetQuery.scala | 10 + .../EvergreenVideosSearchByUserIdsQuery.scala | 5 + ...oricalEvergreenVideosCandidateSource.scala | 44 + .../MemeVideoCandidateSource.scala | 52 + .../SemanticVideoCandidateSource.scala | 57 + ...witterClipV0LongVideoCandidateSource.scala | 53 + ...itterClipV0ShortVideoCandidateSource.scala | 54 + .../candidate_source/ndr_ann/BUILD.bazel | 19 + .../candidate_source/ndr_ann/DRANNKey.scala | 37 + .../ndr_ann/DRMultipleANNQuery.scala | 13 + ...etrievalTweetTweetANNCandidateSource.scala | 116 ++ ...weetTweetEmbeddingANNCandidateSource.scala | 175 ++ ...RetrievalUserTweetANNCandidateSource.scala | 193 ++ .../ndr_ann/EmbeddingANNCandidateSource.scala | 168 ++ ...RetrievalUserTweetANNCandidateSource.scala | 166 ++ .../UserInterestANNCandidateSource.scala | 174 ++ .../popular_geo_tweets/BUILD.bazel | 15 + .../PopularGeoTweetsCandidateSource.scala | 72 + .../TripStratoGeoQuery.scala | 8 + .../popular_grok_topic_tweets/BUILD.bazel | 14 + .../GrokTopicTweetsQuery.scala | 7 + .../PopGrokTopicTweetsCandidateSource.scala | 34 + .../popular_topic_tweets/BUILD.bazel | 14 + .../PopularTopicTweetsCandidateSource.scala | 74 + .../TripStratoTopicQuery.scala | 8 + .../candidate_source/qig_service/BUILD.bazel | 10 + .../QigServiceBatchTweetCandidateSource.scala | 57 + .../qig_service/QigTweetCandidate.scala | 7 + .../simclusters_ann/BUILD.bazel | 19 + .../simclusters_ann/SANNQuery.scala | 9 + .../SimClustersAnnCandidateSource.scala | 111 ++ .../SimclusterColdPostsCandidateSource.scala | 58 + .../SimclusterColdPostsQuery.scala | 7 + ...terPromotedCreatorAnnCandidateSource.scala | 21 + .../text_embedding_ann/BUILD.bazel | 15 + .../TextEmbeddingCandidateSource.scala | 40 + .../TextEmbeddingQuery.scala | 5 + .../candidate_source/topic_tweets/BUILD.bazel | 16 + .../CertoTopicTweetsCandidateSource.scala | 90 + .../SkitTopicTweetsCandidateSource.scala | 78 + .../candidate_source/trends/BUILD.bazel | 16 + .../trends/TrendsCandidateSource.scala | 63 + .../trends/TrendsVideoCandidateSource.scala | 62 + .../candidate_source/twhin_ann/BUILD.bazel | 21 + .../twhin_ann/TwHINANNCandidateSource.scala | 124 ++ .../TwHINRebuildANNCandidateSource.scala | 134 ++ .../twhin_ann/TwHINRebuildANNKey.scala | 10 + .../user_location/BUILD.bazel | 13 + .../UserLocationCandidateSource.scala | 45 + .../candidate_source/uss_service/BUILD.bazel | 12 + .../USSSignalCandidateSource.scala | 34 + .../twitter/tweet_mixer/config/BUILD.bazel | 9 + .../config/SimClustersANNConfig.scala | 199 ++ .../tweet_mixer/config/TimeoutConfig.scala | 17 + .../tweet_mixer/controller/BUILD.bazel | 23 + .../TweetMixerThriftController.scala | 79 + .../twitter/tweet_mixer/feature/BUILD.bazel | 15 + .../tweet_mixer/feature/EntityTypes.scala | 7 + .../feature/FromInNetworkSourceFeature.scala | 6 + .../feature/HydraScoreFeature.scala | 6 + .../feature/InReplyToTweetIdFeature.scala | 6 + .../feature/LanguageCodeFeature.scala | 8 + .../feature/LowSignalUserFeature.scala | 6 + .../feature/MediaMetadataFeatures.scala | 12 + .../feature/PredictionRequestIdFeature.scala | 6 + .../RealGraphInNetworkScoresFeature.scala | 6 + .../RequestCountryPlaceIdFeature.scala | 6 + .../tweet_mixer/feature/ScoreFeature.scala | 6 + .../tweet_mixer/feature/SignalInfo.scala | 11 + .../feature/SourceSignalFeature.scala | 8 + .../feature/SourceTweetIdFeature.scala | 6 + .../tweet_mixer/feature/TopicTweetScore.scala | 6 + .../tweet_mixer/feature/TripTweetScore.scala | 6 + .../feature/TweetInfoFeatures.scala | 48 + .../feature/TweetTopicIdFeature.scala | 6 + .../tweet_mixer/feature/USSFeatures.scala | 217 +++ .../feature/UserTopicIdsFeature.scala | 6 + .../functional_component/BUILD.bazel | 29 + .../TweetMixerFunctionalComponents.scala | 333 ++++ .../functional_component/filter/BUILD.bazel | 18 + .../filter/GrokFilter.scala | 33 + .../filter/ImpressedTweetsBloomFilter.scala | 58 + .../filter/ImpressedTweetsFilter.scala | 38 + .../filter/IsLongFormVideoFilter.scala | 54 + .../filter/IsPortraitVideoFilter.scala | 53 + .../filter/IsShortFormVideoFilter.scala | 55 + .../filter/IsVideoTweetFilter.scala | 61 + .../filter/MaxViewCountFilter.scala | 119 ++ .../filter/MediaClusterIdDedupFilter.scala | 56 + .../filter/MediaIdDedupFilter.scala | 56 + .../filter/MediaWatchHistoryFilter.scala | 48 + .../filter/MinScoreFilter.scala | 80 + ...ShouldIgnoreCandidatePipelinesFilter.scala | 16 + .../TweetVisibilityAndReplyFilter.scala | 62 + .../gate/AllowLowSignalUserGate.scala | 29 + .../AllowNonEmptySearchHistoryUserGate.scala | 24 + .../functional_component/gate/BUILD.bazel | 13 + .../gate/DenyLowSignalUserGate.scala | 27 + .../gate/MaxFollowersGate.scala | 26 + .../gate/MinTimeSinceLastRequestGate.scala | 31 + .../gate/ProbablisticPassGate.scala | 23 + .../functional_component/hydrator/BUILD.bazel | 73 + ...EmbeddingQueryFeatureHydratorFactory.scala | 74 + ...nDRUserEmbeddingQueryFeatureHydrator.scala | 97 + ...EmbeddingQueryFeatureHydratorFactory.scala | 71 + ...AiTopicEmbeddingQueryFeatureHydrator.scala | 64 + ...EmbeddingQueryFeatureHydratorFactory.scala | 179 ++ ...valUserEmbeddingQueryFeatureHydrator.scala | 121 ++ ...nDRUserEmbeddingQueryFeatureHydrator.scala | 97 + .../FeedbackHistoryQueryFeatureHydrator.scala | 58 + .../GizmoduckQueryFeatureHydrator.scala | 45 + .../hydrator/GrokBooleanFeatureHydrator.scala | 72 + .../GrokCategoriesFeatureHydrator.scala | 30 + .../hydrator/GrokFilterFeatureHydrator.scala | 101 + .../HaploliteQueryFeatureHydrator.scala | 91 + ...litySourceSignalQueryFeatureHydrator.scala | 173 ++ ...nkingPreparationQueryFeatureHydrator.scala | 96 + ...ssionBloomFilterQueryFeatureHydrator.scala | 60 + ...VideoBloomFilterVideoFeatureHydrator.scala | 56 + ...stNonPollingTimeQueryFeatureHydrator.scala | 64 + ...EmbeddingQueryFeatureHydratorFactory.scala | 76 + ...valUserEmbeddingQueryFeatureHydrator.scala | 121 ++ ...ediaMetadataCandidateFeatureHydrator.scala | 121 ++ ...EmbeddingQueryFeatureHydratorFactory.scala | 74 + ...EmbeddingQueryFeatureHydratorFactory.scala | 159 ++ ...hInNetworkScoresQueryFeatureHydrator.scala | 60 + ...RequestCountryPlaceIdFeatureHydrator.scala | 54 + ...SGSFollowedUsersQueryFeatureHydrator.scala | 46 + .../SignalInfoCandidateFeatureHydrator.scala | 61 + .../TweetypieCandidateFeatureHydrator.scala | 311 ++++ ...eedTweetsQueryFeatureHydratorFactory.scala | 151 ++ ...ositiveEmbeddingQueryFeatureHydrator.scala | 51 + ...trievalTweetEmbeddingFeatureHydrator.scala | 266 +++ .../USSGrokCategoryFeatureHydrator.scala | 214 +++ .../hydrator/USSQueryFeatureHydrator.scala | 712 +++++++ ...TGOutlierSignalsQueryFeatureHydrator.scala | 94 + ...VGOutlierSignalsQueryFeatureHydrator.scala | 94 + .../UecAggTweetTotalFeatureHydrator.scala | 102 + ...rInterestSummaryQueryFeatureHydrator.scala | 63 + .../UserSignalQueryFeatureHydrator.scala | 81 + .../UserTopicIdsFeatureHydrator.scala | 137 ++ .../functional_component/selector/BUILD.bazel | 19 + .../selector/FavoriteSelector.scala | 33 + .../selector/FeedbackRelevantSelector.scala | 33 + .../selector/HydraBasedSorterProvider.scala | 75 + .../HydraBasedTransformedSorterProvider.scala | 152 ++ ...ndWeightedSignalPriorityWeaveResults.scala | 153 ++ .../selector/ReserveVideoSelector.scala | 40 + .../selector/UprankVideoSorterProvider.scala | 34 + .../side_effect/BUILD.bazel | 31 + .../DeepRetrievalAdHocSideEffect.scala | 146 ++ .../EvergreenVideosSideEffect.scala | 59 + .../side_effect/HydraScoringSideEffect.scala | 76 + .../PublishGroxUserInterestsSideEffect.scala | 49 + ...RequestMultimodalEmbeddingSideEffect.scala | 65 + .../ScribeServedCandidatesSideEffect.scala | 118 ++ .../side_effect/SelectedStatsSideEffect.scala | 142 ++ .../AnnCandidateFeatureTransformer.scala | 31 + .../transformer/BUILD.bazel | 45 + .../CertoTopicTweetsQueryTransformer.scala | 56 + .../EarlybirdInNetworkQueryTransformer.scala | 130 ++ ...dInNetworkResponseFeatureTransformer.scala | 52 + .../EvergreenVideosQueryTransformer.scala | 33 + ...reenVideosResponseFeatureTransformer.scala | 30 + .../GrokTopicTweetsQueryTransformer.scala | 31 + .../HaploliteResponseFeatureTransformer.scala | 45 + .../QigBatchQueryTransformer.scala | 34 + .../QigTweetCandidateFeatureTransformer.scala | 27 + .../transformer/SANNQueryTransformer.scala | 61 + .../SkitTopicTweetsQueryTransformer.scala | 83 + .../TimelineQueryTransformer.scala | 51 + .../TopicTweetFeatureTransformer.scala | 23 + .../TripStratoGeoQueryTransformer.scala | 56 + .../TripStratoTopicQueryTransformer.scala | 76 + .../TripTweetFeatureTransformer.scala | 19 + ...eetFeatureTimelineServiceTransformer.scala | 30 + ...weetMixerCandidateFeatureTransformer.scala | 30 + ...itterClipV0LongVideoQueryTransformer.scala | 32 + ...tterClipV0ShortVideoQueryTransformer.scala | 32 + .../UTGProducerBasedQueryTransformer.scala | 38 + .../UTGTweetBasedQueryTransformer.scala | 67 + .../UVGTweetBasedQueryTransformer.scala | 77 + .../transformer/UtegQueryTransformer.scala | 57 + .../UtegResponseFeatureTransformer.scala | 30 + .../marshaller/request/BUILD.bazel | 27 + .../TweetMixerDebugParamsUnmarshaller.scala | 28 + ...TweetMixerProductContextUnmarshaller.scala | 52 + .../TweetMixerProductUnmarshaller.scala | 34 + .../TweetMixerRequestUnmarshaller.scala | 30 + .../marshaller/response/BUILD.bazel | 36 + .../TweetMixerProductResponseMarshaller.scala | 79 + ...weetMixerResponseTransportMarshaller.scala | 26 + .../marshaller/response/common/BUILD.bazel | 17 + .../common/TweetResultMarshaller.scala | 25 + .../com/twitter/tweet_mixer/model/BUILD.bazel | 8 + .../tweet_mixer/model/ModuleNames.scala | 58 + .../tweet_mixer/model/request/BUILD.bazel | 15 + .../model/request/HasContentCategory.scala | 5 + .../model/request/HasTopicIds.scala | 5 + .../model/request/HasVideoType.scala | 7 + .../request/TweetMixerDebugOptions.scala | 9 + .../model/request/TweetMixerProduct.scala | 55 + .../model/request/TweetMixerRequest.scala | 17 + .../tweet_mixer/model/response/BUILD.bazel | 11 + .../model/response/RecommendationResult.scala | 15 + .../model/response/TweetMixerCandidate.scala | 14 + .../response/TweetMixerProductResponse.scala | 16 + .../model/response/TweetMixerResponse.scala | 15 + .../twitter/tweet_mixer/module/BUILD.bazel | 40 + .../CertoStratoTopicTweetsStoreModule.scala | 58 + .../module/ExtendedStratoClientModule.scala | 70 + .../module/GPURetrievalHttpClientModule.scala | 138 ++ .../module/HaploliteClientModule.scala | 24 + ...beddingGenerationServiceClientModule.scala | 41 + .../module/HydraRootClientModule.scala | 24 + .../module/InMemoryCacheModule.scala | 68 + .../module/MHMtlsParamsModule.scala | 17 + .../ManhattanFeatureRepositoryModule.scala | 82 + .../module/MemCacheClientModule.scala | 34 + .../PipelineFailureExceptionMapper.scala | 29 + ...ureStoreV1DynamicClientBuilderModule.scala | 50 + ...ClustersANNServiceNameToClientMapper.scala | 23 + .../SkitStratoTopicTweetsStoreModule.scala | 62 + .../module/StitchMemcacheClientModule.scala | 20 + .../module/TimeoutConfigModule.scala | 23 + .../module/TwHINANNServiceModule.scala | 33 + .../module/TwHINEmbeddingStoreModule.scala | 54 + .../module/TweetMixerFlagModule.scala | 17 + .../module/UserStateStoreModule.scala | 101 + .../AnnEmbeddingProducerModule.scala | 48 + .../AnnQueryServiceClientModule.scala | 58 + .../AnnQueryableByIdModule.scala | 93 + .../module/thrift_client/BUILD.bazel | 42 + .../EarlybirdRealtimeCGModule.scala | 49 + .../GeoduckHydrationClientModule.scala | 15 + .../GeoduckLocationServiceClientModule.scala | 15 + .../QigServiceClientModule.scala | 29 + .../SimClustersAnnServiceClientModule.scala | 71 + .../thrift_client/TweetyPieClientModule.scala | 56 + .../UserTweetGraphClientModule.scala | 39 + .../UserVideoGraphClientModule.scala | 39 + .../VecDBAnnServiceClientModule.scala | 56 + .../com/twitter/tweet_mixer/param/BUILD.bazel | 15 + .../param/CandidateSourceParams.scala | 30 + .../param/CertoTopicTweetsParams.scala | 59 + .../param/ContentEmbeddingAnnParams.scala | 57 + .../CuratedUserTlsPerLanguageParams.scala | 21 + .../EarlybirdInNetworkTweetsParams.scala | 13 + .../tweet_mixer/param/EvergreenParams.scala | 17 + .../param/GlobalParamConfigModule.scala | 10 + .../param/HighQualitySourceSignalParams.scala | 174 ++ .../param/PopGrokTopicTweetsParams.scala | 27 + .../param/PopularGeoTweetsParams.scala | 48 + .../param/PopularTopicTweetsParams.scala | 51 + .../param/SimClustersAnnParams.scala | 204 ++ .../param/SkitTopicTweetsParams.scala | 93 + .../param/TweetMixerGlobalParamConfig.scala | 82 + .../param/TweetMixerGlobalParams.scala | 1647 +++++++++++++++++ .../twitter/tweet_mixer/param/USSParams.scala | 575 ++++++ .../twitter/tweet_mixer/param/UTGParams.scala | 142 ++ .../twitter/tweet_mixer/param/UVGParams.scala | 176 ++ .../param/UserLocationParams.scala | 13 + .../tweet_mixer/param/decider/BUILD.bazel | 9 + .../param/decider/DeciderKey.scala | 23 + .../twitter/tweet_mixer/product/BUILD.bazel | 19 + .../product/TweetMixerProductModule.scala | 11 + ...etMixerProductPipelineRegistryConfig.scala | 90 + .../home_recommended_tweets/BUILD.bazel | 62 + ...commendedTweetsProductPipelineConfig.scala | 100 + ...edTweetsRecommendationPipelineConfig.scala | 435 +++++ .../marshaller/request/BUILD.bazel | 14 + ...ndedTweetsProductContextUnmarshaller.scala | 21 + .../marshaller/response/BUILD.bazel | 22 + ...mendedTweetsDomainResponseMarshaller.scala | 77 + ...endedTweetsProductResponseMarshaller.scala | 24 + .../model/request/BUILD.bazel | 14 + .../HomeRecommendedTweetsProductContext.scala | 9 + .../request/HomeRecommendedTweetsQuery.scala | 26 + .../model/response/BUILD.bazel | 13 + ...HomeRecommendedTweetsProductResponse.scala | 11 + .../HomeRecommendedTweetsResult.scala | 5 + .../home_recommended_tweets/param/BUILD.bazel | 11 + .../param/HomeRecommendedTweetsParam.scala | 40 + .../HomeRecommendedTweetsParamConfig.scala | 16 + .../twitter/tweet_mixer/scorer/BUILD.bazel | 10 + .../tweet_mixer/scorer/HydraScorer.scala | 95 + .../tweet_mixer/scoring_pipeline/BUILD.bazel | 13 + .../HydraScoringPipelineConfig.scala | 29 + .../twitter/tweet_mixer/service/BUILD.bazel | 11 + .../service/TweetMixerAccessPolicy.scala | 16 + .../TweetMixerNotificationConfig.scala | 93 + .../service/TweetMixerService.scala | 25 + .../com/twitter/tweet_mixer/store/BUILD.bazel | 29 + .../store/TwhinEmbeddingsStore.scala | 139 ++ .../com/twitter/tweet_mixer/utils/BUILD.bazel | 16 + .../utils/BucketSnowflakeIdAgeStats.scala | 58 + .../utils/CandidatePipelineConstants.scala | 64 + .../utils/CandidateSourceUtil.scala | 103 ++ .../utils/ConcurrentMapCache.scala | 12 + .../utils/InjectionTransformer.scala | 43 + .../utils/MemCacheStitchClient.scala | 49 + .../utils/PipelineFailureCategories.scala | 14 + .../tweet_mixer/utils/SignalUtils.scala | 24 + .../tweet_mixer/utils/Transformers.scala | 123 ++ .../com/twitter/tweet_mixer/utils/Utils.scala | 57 + 988 files changed, 65319 insertions(+), 3195 deletions(-) create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/VerifiedPromptCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeHttpController.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/EntryPointPivotModuleDecorator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ForYouTweetCandidateDecorator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/KeywordTrendsModuleCandidateDecorator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/PinnedTweetBroadcastCandidateDecorator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/StoriesModuleCandidateDecorator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TuneFeedModuleCandidateDecorator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TweetCarouselModuleCandidateDecorator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/VideoCarouselModuleCandidateDecorator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/KeywordTrendMetaDescriptionBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/VerifiedPromptBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ChildFeedbackActionBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DebugSocialContextBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetContextBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostDetailsNegativeFeedbackActionBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostFeedbackActionBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostFollowupFeedbackActionBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RelevancePromptCandidateUrtItemBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ServedTypeSocialContextBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TuneFeedFeedbackActionInfoBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AncestorFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorLargeEmbeddingsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BasketballContextFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BroadcastStateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/CategoryDiversityRescoringFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipEmbeddingFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipEmbeddingMediaUnderstandingFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipImageClusterIdFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DependentBulkCandidateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DiversityRescoringFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/EarlybirdSearchResultFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowableUttTopicsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GeoduckAuthorLocationHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GraphTwoHopFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokAnnotationsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokGorkContentCreatorFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokTranslatedPostIsCachedFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/HeartbeatOptimizerParamsHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/HeavyRankerWeightsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressedImageClusterIdsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressedMediaClusterIdsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNegativeFeedbackTimeQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ListIdsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MediaClusterIdFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MediaCompletionRateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MultiModalEmbeddingsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NaviClientConfigQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NaviVideoClientConfigQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OnPremRealGraphQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OptimizerWeightsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OriginalAuthorLargeEmbeddingsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OriginalTweetLargeEmbeddingsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PhoenixRescoringFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PostContextFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RateLimitQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeEntityRealGraphQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestTimeQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersLogFavBasedTweetFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersUserSparseEmbeddingsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersUserTweetScoresHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimclusterBasedTopAuthorsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SlopAuthorFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SpaceStateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TransformerPostEmbeddingFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetEntityServiceContentFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetEntityServiceFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetLanguageFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetLargeEmbeddingsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetMetaDataFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetTimeFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetTypeMetricsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildTweetFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildUserEngagementQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildUserPositiveQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinTweetFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserNegativeFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserPositiveFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinVideoFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UnifiedUserActionsUserIdentifierFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserActionByteArrayQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserActionsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagedGrokCategoriesFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagedLanguagesFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagementGrokTagFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFrequentLocationHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFrequentLocationQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserHistoryTransformerEmbeddingQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLanguagesFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLargeEmbeddingsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserStateQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserSubscriptionQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserUnderstandableLangaugesFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UtegFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/VideoSummaryEmbeddingFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ViewCountsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ViralContentCreatorMetricsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/WithDefaultFeatureMap.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ClipEmbeddingFeaturesAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ContentFeatureAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/TextTokensFeaturesAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/VideoSummaryEmbeddingFeaturesAdaptor.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features/GizmoduckFeaturesAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features/LightRankingCandidateFeaturesAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features/SimclustersFeaturesAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/TransformerEmbeddingsAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/UserHistoryEventsAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/VideoUserHistoryEventsAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TopicEdgeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TopicEdgeTruncatedAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TweetContentEdgeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/UserEngagerEdgeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/UserEntityEdgeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Utils.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserTweetTvVideoRealTimeAggregateFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/BaseUserHistoryEventsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/ScoredTweetsUserHistoryEventsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/ScoredVideoTweetsUserHistoryEventsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/AuthorDedupFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ClipClusterDeduplicationFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ClusterBasedDedupFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ConsistentAspectRatioFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/CountryFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/CurrentPinnedTweetFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokGoreFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokNsfwFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokSpamFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokViolentFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/HasAuthorFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/HasMultipleMediaFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/LocationFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MaxVideoDurationFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MediaDeduplicationFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MinVideoDurationFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenMediaIdsFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/QuoteDeduplicationFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RegionFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/SlopFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/TweetHydrationFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/WeeklyBookmarkFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/AllowForYouRecommendationsGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BookmarksTimeGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSyntheticUserGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/PersistenceStoreDurationValidationGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RateLimitGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RateLimitNotGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RecentlyServedByServedTypeGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TestUserProbabilisticGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/NaviModelScorer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PhoenixModelRerankingScorer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PhoenixScorer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PredictClientFactory.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/WeighedModelRerankingScorer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/RandomShuffleCandidates.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/ScoreAveragingPositionSelector.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/SortFixedPositionCandidates.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BaseCacheCandidateFeaturesSideEffect.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/CommonFeaturesPldrConverter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GrokTopics.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeLargeEmbeddingsFeatures.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/model/NaviClientConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/model/NavigationIncludeInstruction.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/model/PhoenixPredictedScoreFeature.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/model/PredictedScoreFeature.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source/SourceSignal.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup/SignupSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClusterDetailsModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/EarlybirdRealtimeCGModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/EventsRecosClientModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/GizmoduckTimelinesCacheClientModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFeaturesModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InMemoryCacheModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/LimiterModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MediaClusterId88Module.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MediaClusterIdModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedScoredCandidateFeaturesStoreModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PhoenixClientModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredVideoTweetsMemcacheModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TvWatchHistoryCacheClientModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetWatchTimeMetadataModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TwhinEmbeddingsModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/UttTopicModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/module/VideoEmbeddingModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsDependentCandidatePipelineBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingDependentAdsMixerPipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouBookmarksCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouCommunitiesToJoinCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouEntryPointPivotCandidatePipelineBuilder.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouEntryPointPivotCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouExplorationTweetsCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouJetfuelFrameFrameCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouKeywordTrendsCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouMixerPipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPinnedTweetsCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRecommendedJobsCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRelevancePromptCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouResponseDomainMarshaller.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredVideoTweetsCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouStoriesCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTuneFeedCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BookmarksCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BroadcastedPinnedTweetsCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/JetfuelFrameCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/RecommendedJobsCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/RecommendedRecruitingOrganizationsCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredVideoTweetsCategorizedProductCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredVideoTweetsProductCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/StoriesModuleCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/TuneFeedCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/UnifiedTrendsCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/ArticlePreviewTextFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/CurrentPinnedTweetFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/DisplayedGrokTopicQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FollowingSportsAccountQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetAuthorFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetAuthorFollowersFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetEngagementsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/ViewerHasJobRecommendationsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/NotArticleFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/PromotedTrendFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/FollowingSportsUsersGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/TuneFeedModuleGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/UserFollowingRangeGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/UnifiedCandidatesQueryTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BookmarksResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/ExplorationTweetResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/KeywordTrendsFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/PinnedTweetResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/ScoredVideoTweetResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/StoriesModuleResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TuneFeedFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/PinnedTweetCandidateScorer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/DebugUpdateSortAdsResult.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/DebunchCandidates.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/RemoveDuplicateCandidatesOutsideModule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/VideoServedStatsSideEffect.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsContentExplorationCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsDirectUtegCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsStaticCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsCommunitiesCandidatePipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ContentExplorationCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/EarlybirdRealtimeCGTweetCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/StaticPostsCandidateSource.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FollowedUserScoresFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/InvalidateCachedScoredTweetsQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsColdStartPostFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListNameFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/LowSignalUserQueryFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SGSMutuallyFollowedUserHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SemanticCoreFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieVisibilityFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ValidLikedByUserIdsFeatureHydrator.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ControlAiExcludeFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ControlAiOnlyIncludeFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilterWithIncludeRule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilterWithSkipRule.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ExtendedDirectedAtFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/FollowedAuthorFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/GrokAutoTranslateLanguageFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/IsOutOfNetworkColdStartPostFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/LanguageFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OONReplyFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/QualifiedRepliesFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/SGSAuthorFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/TopKFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/TopKOptionalFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/UtegMinFavCountFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/UtegTopKFilter.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/AllowLowSignalUserGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/DenyLowSignalUserGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MatchesCountryGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/RecentFeedbackCheckGate.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/ContentExplorationQueryTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/UtegQueryTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/CommunitiesEarlybirdQueryTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsContentExplorationResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsDirectUtegResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsOfflineVideoRecoResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsStaticResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredVideoTweetsPinnedTweetResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsCommunitiesResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdNsfwResponseFeatureTransformer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/AuthorBasedListwiseRescoringProvider.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/CandidateSourceDiversityListwiseRescoringProvider.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ContentExplorationListwiseRescoringProvider.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ControlAiRescorer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DeepRetrievalListwiseRescoringProvider.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/EvergreenDeepRetrievalCrossBorderListwiseRescoringProvider.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/EvergreenDeepRetrievalListwiseRescoringProvider.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/GrokSlopScoreRescorer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedAuthorDecayRescoringProvider.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedImageClusterBasedListwiseRescoringProvider.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedMediaClusterBasedListwiseRescoringProvider.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ListwiseRescoringProvider.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/LowSignalScorer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/MultimodalEmbeddingRescorer.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsLowSignalScoringPipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsRerankingScoringPipelineConfig.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepTopKCandidatesPerCommunity.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheCandidateFeaturesSideEffect.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheRequestInfoSideEffect.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheRetrievalSignalSideEffect.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CommonFeaturesSideEffect.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredCandidateFeatureKeysKafkaSideEffect.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredPhoenixCandidatesKafkaSideEffect.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredStatsSideEffect.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredTweetsDiversityStatsSideEffect.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util/BUILD.bazel create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util/ControlAiUtil.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterId88Store.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterId95Store.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterIdStoreTrait.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RTAMHStore.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/store/TweetWatchTimeMetadataStore.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/store/TwhinEmbeddingsStore.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/store/VideoEmbeddingMHStore.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageCode.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/util/NaviScorerStatsHandler.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/util/PhoenixScorerStatsHandler.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/util/PhoenixUtils.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/util/RerankerUtil.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/util/SignalUtil.scala create mode 100644 home-mixer/server/src/main/scala/com/twitter/home_mixer/util/UrtUtil.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerHttpServerWarmupHandler.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerServer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerThriftServerWarmupHandler.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/CertoTopicTweetsCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentAnnTweetBasedCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRTweetTweetCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRTweetTweetTierTwoCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRUserTweetCandidatePipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRUserTweetTierTwoCandidatePipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEmbeddingSimilarityCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEvergreenDRTweetTweetCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationSimclusterColdCandidatePipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ControlAiTopicCandidatePipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/CuratedUserTlsPerLangaugeCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalUserTweetSimilarityCandidatePipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EarlybirdInNetworkCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EventsCandidatePipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenDRCrossBorderUserTweetCandidatePipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenDRUserTweetCandidatePipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenVideosCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/HaploliteCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/MediaDeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PinnedTweetRelatedCreatorPipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopGrokTopicTweetsCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopularGeoTweetsCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopularTopicTweetsCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/QigSearchHistoryTweetsCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SemanticVideoCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersInterestedInCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersProducerBasedCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersPromotedCreatorCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersTweetBasedCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SkitTopicTweetsCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TrendsVideoCandidatePipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwHINRebuildTweetSimilarityCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwHINTweetSimilarityCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwhinConsumerBasedCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwhinUserTweetSimilarityCandidatePipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwitterClipV0LongVideoCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwitterClipV0ShortVideoCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTEGCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGExpansionTweetBasedCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGProducerBasedCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGTweetBasedCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UVGExpansionTweetBasedCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UVGTweetBasedCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UserInterestsSummaryCandidatePipelineConfigFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UserLocationCandidatePipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UTGProducerBasedRequest.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UTGTweetBasedRequest.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphConsumerBasedCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphProducerBasedCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphTweetBasedCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UVGTweetBasedRequest.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UserVideoGraphConsumerBasedCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UserVideoGraphTweetBasedCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source/MemcachedCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/content_embedding_ann/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/content_embedding_ann/ContentEmbeddingAnnCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/curated_user_tls_per_language/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/curated_user_tls_per_language/CuratedUserTlsPerLanguageCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/EarlybirdRealtimeCGTweetCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/InNetworkRequest.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/engaged_users/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/engaged_users/RecentEngagedUsersCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/events/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/events/EventsCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/EvergreenVideosSearchByTweetQuery.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/EvergreenVideosSearchByUserIdsQuery.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/HistoricalEvergreenVideosCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/MemeVideoCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/SemanticVideoCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/TwitterClipV0LongVideoCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/TwitterClipV0ShortVideoCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DRANNKey.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DRMultipleANNQuery.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalTweetTweetANNCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalTweetTweetEmbeddingANNCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalUserTweetANNCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/EmbeddingANNCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/MediaDeepRetrievalUserTweetANNCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/UserInterestANNCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/PopularGeoTweetsCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/TripStratoGeoQuery.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/GrokTopicTweetsQuery.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/PopGrokTopicTweetsCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/PopularTopicTweetsCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/TripStratoTopicQuery.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/QigServiceBatchTweetCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/QigTweetCandidate.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SANNQuery.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimClustersAnnCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterColdPostsCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterColdPostsQuery.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterPromotedCreatorAnnCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/TextEmbeddingCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/TextEmbeddingQuery.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/CertoTopicTweetsCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/SkitTopicTweetsCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/TrendsCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/TrendsVideoCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINANNCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINRebuildANNCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINRebuildANNKey.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/user_location/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/user_location/UserLocationCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/uss_service/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/uss_service/USSSignalCandidateSource.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/SimClustersANNConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/TimeoutConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/controller/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/controller/TweetMixerThriftController.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/EntityTypes.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/FromInNetworkSourceFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/HydraScoreFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/InReplyToTweetIdFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/LanguageCodeFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/LowSignalUserFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/MediaMetadataFeatures.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/PredictionRequestIdFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/RealGraphInNetworkScoresFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/RequestCountryPlaceIdFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/ScoreFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SignalInfo.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SourceSignalFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SourceTweetIdFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TopicTweetScore.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TripTweetScore.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TweetInfoFeatures.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TweetTopicIdFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/USSFeatures.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/UserTopicIdsFeature.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/TweetMixerFunctionalComponents.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/GrokFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ImpressedTweetsBloomFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ImpressedTweetsFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsLongFormVideoFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsPortraitVideoFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsShortFormVideoFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsVideoTweetFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MaxViewCountFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaClusterIdDedupFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaIdDedupFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaWatchHistoryFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MinScoreFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ShouldIgnoreCandidatePipelinesFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/TweetVisibilityAndReplyFilter.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/AllowLowSignalUserGate.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/AllowNonEmptySearchHistoryUserGate.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/DenyLowSignalUserGate.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/MaxFollowersGate.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/MinTimeSinceLastRequestGate.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/ProbablisticPassGate.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentEmbeddingQueryFeatureHydratorFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentExplorationDRUserEmbeddingQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentMediaEmbeddingQueryFeatureHydratorFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ControlAiTopicEmbeddingQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/DeepRetrievalUserEmbeddingQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/EvergreenDRUserEmbeddingQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/FeedbackHistoryQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GizmoduckQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokBooleanFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokCategoriesFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokFilterFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HaploliteQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HighQualitySourceSignalQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HydraRankingPreparationQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ImpressionVideoBloomFilterVideoFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/LastNonPollingTimeQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaDeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaDeepRetrievalUserEmbeddingQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaMetadataCandidateFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MultimodalEmbeddingQueryFeatureHydratorFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/RequestCountryPlaceIdFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/SGSFollowedUsersQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/SignalInfoCandidateFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TweetypieCandidateFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TweetypieSeedTweetsQueryFeatureHydratorFactory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TwhinUserPositiveEmbeddingQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSDeepRetrievalTweetEmbeddingFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSGrokCategoryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UTGOutlierSignalsQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UVGOutlierSignalsQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UecAggTweetTotalFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserInterestSummaryQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserSignalQueryFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserTopicIdsFeatureHydrator.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/FavoriteSelector.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/FeedbackRelevantSelector.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/HydraBasedSorterProvider.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/HydraBasedTransformedSorterProvider.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/InsertAppendWeightedSignalPriorityWeaveResults.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/ReserveVideoSelector.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/UprankVideoSorterProvider.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/DeepRetrievalAdHocSideEffect.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/EvergreenVideosSideEffect.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/HydraScoringSideEffect.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/PublishGroxUserInterestsSideEffect.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/RequestMultimodalEmbeddingSideEffect.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/ScribeServedCandidatesSideEffect.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/SelectedStatsSideEffect.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/AnnCandidateFeatureTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/CertoTopicTweetsQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EarlybirdInNetworkQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EarlybirdInNetworkResponseFeatureTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EvergreenVideosQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EvergreenVideosResponseFeatureTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/GrokTopicTweetsQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/HaploliteResponseFeatureTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/QigBatchQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/QigTweetCandidateFeatureTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/SANNQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/SkitTopicTweetsQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TimelineQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TopicTweetFeatureTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripStratoGeoQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripStratoTopicQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripTweetFeatureTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TweetFeatureTimelineServiceTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TweetMixerCandidateFeatureTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TwitterClipV0LongVideoQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TwitterClipV0ShortVideoQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UTGProducerBasedQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UTGTweetBasedQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UVGTweetBasedQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UtegQueryTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UtegResponseFeatureTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerDebugParamsUnmarshaller.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerProductContextUnmarshaller.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerProductUnmarshaller.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerRequestUnmarshaller.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/TweetMixerProductResponseMarshaller.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/TweetMixerResponseTransportMarshaller.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/common/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/common/TweetResultMarshaller.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/ModuleNames.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasContentCategory.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasTopicIds.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasVideoType.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerDebugOptions.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerProduct.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerRequest.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/RecommendationResult.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerCandidate.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerProductResponse.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerResponse.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/CertoStratoTopicTweetsStoreModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/ExtendedStratoClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/GPURetrievalHttpClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HaploliteClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HydraEmbeddingGenerationServiceClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HydraRootClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/InMemoryCacheModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/MHMtlsParamsModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/ManhattanFeatureRepositoryModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/MemCacheClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/PipelineFailureExceptionMapper.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SampleFeatureStoreV1DynamicClientBuilderModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SimClustersANNServiceNameToClientMapper.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SkitStratoTopicTweetsStoreModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/StitchMemcacheClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TimeoutConfigModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TwHINANNServiceModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TwHINEmbeddingStoreModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TweetMixerFlagModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/UserStateStoreModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnEmbeddingProducerModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnQueryServiceClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnQueryableByIdModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/EarlybirdRealtimeCGModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/GeoduckHydrationClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/GeoduckLocationServiceClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/QigServiceClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/SimClustersAnnServiceClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/TweetyPieClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/UserTweetGraphClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/UserVideoGraphClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/VecDBAnnServiceClientModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CandidateSourceParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CertoTopicTweetsParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/ContentEmbeddingAnnParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CuratedUserTlsPerLanguageParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/EarlybirdInNetworkTweetsParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/EvergreenParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/GlobalParamConfigModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/HighQualitySourceSignalParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopGrokTopicTweetsParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopularGeoTweetsParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopularTopicTweetsParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/SimClustersAnnParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/SkitTopicTweetsParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/TweetMixerGlobalParamConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/TweetMixerGlobalParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/USSParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UTGParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UVGParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UserLocationParams.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/decider/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/decider/DeciderKey.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/TweetMixerProductModule.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/TweetMixerProductPipelineRegistryConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/HomeRecommendedTweetsProductPipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/HomeRecommendedTweetsRecommendationPipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/request/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/request/HomeRecommendedTweetsProductContextUnmarshaller.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/HomeRecommendedTweetsDomainResponseMarshaller.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/HomeRecommendedTweetsProductResponseMarshaller.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/HomeRecommendedTweetsProductContext.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/HomeRecommendedTweetsQuery.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/HomeRecommendedTweetsProductResponse.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/HomeRecommendedTweetsResult.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/HomeRecommendedTweetsParam.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/HomeRecommendedTweetsParamConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scorer/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scorer/HydraScorer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scoring_pipeline/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scoring_pipeline/HydraScoringPipelineConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerAccessPolicy.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerNotificationConfig.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerService.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/store/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/store/TwhinEmbeddingsStore.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/BUILD.bazel create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/BucketSnowflakeIdAgeStats.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/CandidatePipelineConstants.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/CandidateSourceUtil.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/ConcurrentMapCache.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/InjectionTransformer.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/MemCacheStitchClient.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/PipelineFailureCategories.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/SignalUtils.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/Transformers.scala create mode 100644 tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/Utils.scala diff --git a/README.md b/README.md index b872faef5..102c4bc06 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,25 @@ -# Twitter's Recommendation Algorithm +# X's Recommendation Algorithm -Twitter's Recommendation Algorithm is a set of services and jobs that are responsible for serving feeds of Tweets and other content across all Twitter product surfaces (e.g. For You Timeline, Search, Explore, Notifications). For an introduction to how the algorithm works, please refer to our [engineering blog](https://blog.twitter.com/engineering/en_us/topics/open-source/2023/twitter-recommendation-algorithm). +X's Recommendation Algorithm is a set of services and jobs that are responsible for serving feeds of posts and other content across all X product surfaces (e.g. For You Timeline, Search, Explore, Notifications). For an introduction to how the algorithm works, please refer to our [engineering blog](https://blog.x.com/engineering/en_us/topics/open-source/2023/twitter-recommendation-algorithm). ## Architecture -Product surfaces at Twitter are built on a shared set of data, models, and software frameworks. The shared components included in this repository are listed below: +Product surfaces at X are built on a shared set of data, models, and software frameworks. The shared components included in this repository are listed below: | Type | Component | Description | |------------|------------|------------| -| Data | [tweetypie](tweetypie/server/README.md) | Core Tweet service that handles the reading and writing of Tweet data. | -| | [unified-user-actions](unified_user_actions/README.md) | Real-time stream of user actions on Twitter. | +| Data | [tweetypie](tweetypie/server/README.md) | Core service that handles the reading and writing of post data. | +| | [unified-user-actions](unified_user_actions/README.md) | Real-time stream of user actions on X. | | | [user-signal-service](user-signal-service/README.md) | Centralized platform to retrieve explicit (e.g. likes, replies) and implicit (e.g. profile visits, tweet clicks) user signals. | | Model | [SimClusters](src/scala/com/twitter/simclusters_v2/README.md) | Community detection and sparse embeddings into those communities. | -| | [TwHIN](https://github.com/twitter/the-algorithm-ml/blob/main/projects/twhin/README.md) | Dense knowledge graph embeddings for Users and Tweets. | +| | [TwHIN](https://github.com/twitter/the-algorithm-ml/blob/main/projects/twhin/README.md) | Dense knowledge graph embeddings for Users and Posts. | | | [trust-and-safety-models](trust_and_safety_models/README.md) | Models for detecting NSFW or abusive content. | -| | [real-graph](src/scala/com/twitter/interaction_graph/README.md) | Model to predict the likelihood of a Twitter User interacting with another User. | -| | [tweepcred](src/scala/com/twitter/graph/batch/job/tweepcred/README) | Page-Rank algorithm for calculating Twitter User reputation. | +| | [real-graph](src/scala/com/twitter/interaction_graph/README.md) | Model to predict the likelihood of an X User interacting with another User. | +| | [tweepcred](src/scala/com/twitter/graph/batch/job/tweepcred/README) | Page-Rank algorithm for calculating X User reputation. | | | [recos-injector](recos-injector/README.md) | Streaming event processor for building input streams for [GraphJet](https://github.com/twitter/GraphJet) based services. | -| | [graph-feature-service](graph-feature-service/README.md) | Serves graph features for a directed pair of Users (e.g. how many of User A's following liked Tweets from User B). | -| | [topic-social-proof](topic-social-proof/README.md) | Identifies topics related to individual Tweets. | -| | [representation-scorer](representation-scorer/README.md) | Compute scores between pairs of entities (Users, Tweets, etc.) using embedding similarity. | +| | [graph-feature-service](graph-feature-service/README.md) | Serves graph features for a directed pair of users (e.g. how many of User A's following liked posts from User B). | +| | [topic-social-proof](topic-social-proof/README.md) | Identifies topics related to individual posts. | +| | [representation-scorer](representation-scorer/README.md) | Compute scores between pairs of entities (Users, Posts, etc.) using embedding similarity. | | Software framework | [navi](navi/README.md) | High performance, machine learning model serving written in Rust. | | | [product-mixer](product-mixer/README.md) | Software framework for building feeds of content. | | | [timelines-aggregation-framework](timelines/data_processing/ml_util/aggregation_framework/README.md) | Framework for generating aggregate features in batch or real time. | @@ -38,15 +38,15 @@ The core components of the For You Timeline included in this repository are list | Type | Component | Description | |------------|------------|------------| -| Candidate Source | [search-index](src/java/com/twitter/search/README.md) | Find and rank In-Network Tweets. ~50% of Tweets come from this candidate source. | -| | [cr-mixer](cr-mixer/README.md) | Coordination layer for fetching Out-of-Network tweet candidates from underlying compute services. | -| | [user-tweet-entity-graph](src/scala/com/twitter/recos/user_tweet_entity_graph/README.md) (UTEG)| Maintains an in memory User to Tweet interaction graph, and finds candidates based on traversals of this graph. This is built on the [GraphJet](https://github.com/twitter/GraphJet) framework. Several other GraphJet based features and candidate sources are located [here](src/scala/com/twitter/recos). | -| | [follow-recommendation-service](follow-recommendations-service/README.md) (FRS)| Provides Users with recommendations for accounts to follow, and Tweets from those accounts. | -| Ranking | [light-ranker](src/python/twitter/deepbird/projects/timelines/scripts/models/earlybird/README.md) | Light Ranker model used by search index (Earlybird) to rank Tweets. | -| | [heavy-ranker](https://github.com/twitter/the-algorithm-ml/blob/main/projects/home/recap/README.md) | Neural network for ranking candidate tweets. One of the main signals used to select timeline Tweets post candidate sourcing. | -| Tweet mixing & filtering | [home-mixer](home-mixer/README.md) | Main service used to construct and serve the Home Timeline. Built on [product-mixer](product-mixer/README.md). | -| | [visibility-filters](visibilitylib/README.md) | Responsible for filtering Twitter content to support legal compliance, improve product quality, increase user trust, protect revenue through the use of hard-filtering, visible product treatments, and coarse-grained downranking. | -| | [timelineranker](timelineranker/README.md) | Legacy service which provides relevance-scored tweets from the Earlybird Search Index and UTEG service. | +| Candidate Source | [search-index](src/java/com/twitter/search/README.md) | Find and rank In-Network posts. ~50% of posts come from this candidate source. | +| | [tweet-mixer](tweet-mixer) | Coordination layer for fetching Out-of-Network tweet candidates from underlying compute services. | +| | [user-tweet-entity-graph](src/scala/com/twitter/recos/user_tweet_entity_graph/README.md) (UTEG)| Maintains an in memory User to Post interaction graph, and finds candidates based on traversals of this graph. This is built on the [GraphJet](https://github.com/twitter/GraphJet) framework. Several other GraphJet based features and candidate sources are located [here](src/scala/com/twitter/recos). | +| | [follow-recommendation-service](follow-recommendations-service/README.md) (FRS)| Provides Users with recommendations for accounts to follow, and posts from those accounts. | +| Ranking | [light-ranker](src/python/twitter/deepbird/projects/timelines/scripts/models/earlybird/README.md) | Light Ranker model used by search index (Earlybird) to rank posts. | +| | [heavy-ranker](https://github.com/twitter/the-algorithm-ml/blob/main/projects/home/recap/README.md) | Neural network for ranking candidate posts. One of the main signals used to select timeline posts post candidate sourcing. | +| Post mixing & filtering | [home-mixer](home-mixer/README.md) | Main service used to construct and serve the Home Timeline. Built on [product-mixer](product-mixer/README.md). | +| | [visibility-filters](visibilitylib/README.md) | Responsible for filtering X content to support legal compliance, improve product quality, increase user trust, protect revenue through the use of hard-filtering, visible product treatments, and coarse-grained downranking. | +| | [timelineranker](timelineranker/README.md) | Legacy service which provides relevance-scored posts from the Earlybird Search Index and UTEG service. | ### Recommended Notifications @@ -54,8 +54,8 @@ The core components of Recommended Notifications included in this repository are | Type | Component | Description | |------------|------------|------------| -| Service | [pushservice](pushservice/README.md) | Main recommendation service at Twitter used to surface recommendations to our users via notifications. -| Ranking | [pushservice-light-ranker](pushservice/src/main/python/models/light_ranking/README.md) | Light Ranker model used by pushservice to rank Tweets. Bridges candidate generation and heavy ranking by pre-selecting highly-relevant candidates from the initial huge candidate pool. | +| Service | [pushservice](pushservice/README.md) | Main recommendation service at X used to surface recommendations to our users via notifications. +| Ranking | [pushservice-light-ranker](pushservice/src/main/python/models/light_ranking/README.md) | Light Ranker model used by pushservice to rank posts. Bridges candidate generation and heavy ranking by pre-selecting highly-relevant candidates from the initial huge candidate pool. | | | [pushservice-heavy-ranker](pushservice/src/main/python/models/heavy_ranking/README.md) | Multi-task learning model to predict the probabilities that the target users will open and engage with the sent notifications. | ## Build and test code @@ -64,6 +64,6 @@ We include Bazel BUILD files for most components, but not a top-level BUILD or W ## Contributing -We invite the community to submit GitHub issues and pull requests for suggestions on improving the recommendation algorithm. We are working on tools to manage these suggestions and sync changes to our internal repository. Any security concerns or issues should be routed to our official [bug bounty program](https://hackerone.com/twitter) through HackerOne. We hope to benefit from the collective intelligence and expertise of the global community in helping us identify issues and suggest improvements, ultimately leading to a better Twitter. +We invite the community to submit GitHub issues and pull requests for suggestions on improving the recommendation algorithm. We are working on tools to manage these suggestions and sync changes to our internal repository. Any security concerns or issues should be routed to our official [bug bounty program](https://hackerone.com/x) through HackerOne. We hope to benefit from the collective intelligence and expertise of the global community in helping us identify issues and suggest improvements, ultimately leading to a better X. -Read our blog on the open source initiative [here](https://blog.twitter.com/en_us/topics/company/2023/a-new-era-of-transparency-for-twitter). +Read our blog on the open source initiative [here](https://blog.x.com/en_us/topics/company/2023/a-new-era-of-transparency-for-twitter). diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel index 103e079da..bcf2b543e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel @@ -4,21 +4,8 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-core/src/main", - "finagle/finagle-http/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", + "finagle/finagle-netty4/src/main/scala", "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/http-core/src/main/java/com/twitter/finatra/http", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-utils/src/main/scala", "home-mixer/server/src/main/resources", "home-mixer/server/src/main/scala/com/twitter/home_mixer/controller", "home-mixer/server/src/main/scala/com/twitter/home_mixer/federated", @@ -26,26 +13,12 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product", "home-mixer/thrift/src/main/thrift:thrift-scala", + "joinkey/src/main/scala/com/twitter/joinkey/context", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "strato/config/columns/auth-context:auth-context-strato-client", - "strato/config/columns/gizmoduck:gizmoduck-strato-client", - "strato/src/main/scala/com/twitter/strato/fed", "strato/src/main/scala/com/twitter/strato/fed/server", - "stringcenter/client", - "stringcenter/client/src/main/java", - "stringcenter/client/src/main/scala/com/twitter/stringcenter/client", - "thrift-web-forms/src/main/scala/com/twitter/thriftwebforms/view", "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/features/app", - "twitter-server-internal", - "twitter-server/server/src/main/scala", - "util/util-app/src/main/scala", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala index e635c7a68..07b20a610 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala @@ -1,7 +1,15 @@ package com.twitter.home_mixer import com.google.inject.Module +import com.twitter.conversions.DurationOps._ +import com.twitter.conversions.StorageUnitOps.richStorageUnitFromInt import com.twitter.finagle.Filter +import com.twitter.finagle.Http +import com.twitter.finagle.Thrift +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.mtls.server.MtlsStackServer.MtlsHttpServerSyntax +import com.twitter.finagle.netty4.param.TrackWorkerPool import com.twitter.finatra.annotations.DarkTrafficFilterType import com.twitter.finatra.http.HttpServer import com.twitter.finatra.http.routing.HttpRouter @@ -11,29 +19,37 @@ import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule import com.twitter.finatra.thrift.ThriftServer import com.twitter.finatra.thrift.filters._ import com.twitter.finatra.thrift.routing.ThriftRouter +import com.twitter.home_mixer.controller.HomeHttpController import com.twitter.home_mixer.controller.HomeThriftController import com.twitter.home_mixer.federated.HomeMixerColumn import com.twitter.home_mixer.module._ import com.twitter.home_mixer.param.GlobalParamConfigModule import com.twitter.home_mixer.product.HomeMixerProductModule import com.twitter.home_mixer.{thriftscala => st} +import com.twitter.joinkey.context.CreateRequestJoinKeyContextFilter import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule +import com.twitter.product_mixer.component_library.module.CommunitiesMixerClientModule import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule import com.twitter.product_mixer.component_library.module.EarlybirdModule -import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule +import com.twitter.product_mixer.component_library.module.FeedbackHistoryClientModule import com.twitter.product_mixer.component_library.module.GizmoduckClientModule +import com.twitter.product_mixer.component_library.module.MemcachedImpressionBloomFilterStoreModule import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule +import com.twitter.product_mixer.component_library.module.StaleTweetsCacheModule +import com.twitter.product_mixer.component_library.module.TestUserMapperConfigModule import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule -import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule import com.twitter.product_mixer.component_library.module.TweetMixerClientModule import com.twitter.product_mixer.component_library.module.UserSessionStoreModule +import com.twitter.product_mixer.component_library.module.UtegClientModule +import com.twitter.product_mixer.component_library.module.UtvgClientModule import com.twitter.product_mixer.core.controllers.ProductMixerController import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper import com.twitter.product_mixer.core.module.ProductMixerModule import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule +import com.twitter.scrooge.TUnsafeBinaryProtocolFactory import com.twitter.strato.fed.StratoFed import com.twitter.strato.fed.server.StratoFedServer @@ -50,52 +66,71 @@ class HomeMixerServer override val modules: Seq[Module] = Seq( AccountRecommendationsMixerModule, AdvertiserBrandSafetySettingsStoreModule, - BlenderClientModule, ClientSentImpressionsPublisherModule, + ClusterDetailsModule, + CommunitiesMixerClientModule, ConversationServiceModule, EarlybirdModule, - ExploreRankerClientModule, + EarlybirdRealtimeCGModule, + EventsRecosClientModule, FeedbackHistoryClientModule, GizmoduckClientModule, + GizmoduckTimelinesCacheClientModule, GlobalParamConfigModule, HomeAdsCandidateSourceModule, + HomeMixerFeaturesModule, HomeMixerFlagsModule, HomeMixerProductModule, HomeMixerResourcesModule, - ImpressionBloomFilterModule, + InMemoryCacheModule, InjectionHistoryClientModule, + LimiterModule, ManhattanClientsModule, ManhattanFeatureRepositoryModule, - ManhattanTweetImpressionStoreModule, + MediaClusterId88Module, + MediaClusterId95Module, MemcachedFeatureRepositoryModule, + MemcachedImpressionBloomFilterStoreModule, + MemcachedScoredCandidateFeaturesStoreModule, NaviModelClientModule, OnboardingTaskServiceModule, OptimizedStratoClientModule, PeopleDiscoveryServiceModule, + PhoenixClientModule, ProductMixerModule, RealGraphInNetworkScoresModule, RealtimeAggregateFeatureRepositoryModule, ScoredTweetsMemcacheModule, + ScoredVideoTweetsMemcacheModule, ScribeEventPublisherModule, SimClustersRecentEngagementsClientModule, SocialGraphServiceModule, StaleTweetsCacheModule, + TestUserMapperConfigModule, ThriftFeatureRepositoryModule, TimelineRankerClientModule, - TimelineScorerClientModule, TimelineServiceClientModule, TimelinesPersistenceStoreClientModule, TopicSocialProofClientModule, + TvWatchHistoryCacheClientModule, TweetImpressionStoreModule, TweetMixerClientModule, + TweetWatchTimeMetadataModule, TweetypieClientModule, TweetypieStaticEntitiesCacheClientModule, + TwhinEmbeddingsModule, UserSessionStoreModule, + UtegClientModule, + UttTopicModule, + UtvgClientModule, + VideoEmbeddingModule, new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](), new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this), new ProductScopeStringCenterModule() ) + val requestJoinKeyContextFilter = new CreateRequestJoinKeyContextFilter + override def configureThrift(router: ThriftRouter): Unit = { router .filter[LoggingMDCFilter] @@ -105,16 +140,38 @@ class HomeMixerServer .filter[AccessLoggingFilter] .filter[ExceptionMappingFilter] .filter[Filter.TypeAgnostic, DarkTrafficFilterType] + .filter(requestJoinKeyContextFilter) .exceptionMapper[LoggingThrowableExceptionMapper] .exceptionMapper[PipelineFailureExceptionMapper] .add[HomeThriftController] } + override def configureStratoThriftServer(server: ThriftMux.Server): ThriftMux.Server = { + super + .configureStratoThriftServer(server) + .configured( + TrackWorkerPool( + enableTracking = true, + trackingTaskPeriod = 20.milliseconds, + threadDumpThreshold = 0.milliseconds + )) + .withMaxReusableBufferSize(1.megabyte.bytes.toInt) + .withProtocolFactory(new TUnsafeBinaryProtocolFactory(Thrift.param.protocolFactory)) + } + override def configureHttp(router: HttpRouter): Unit = - router.add( - ProductMixerController[st.HomeMixer.MethodPerEndpoint]( - this.injector, - st.HomeMixer.ExecutePipeline)) + router + .add( + ProductMixerController[st.HomeMixer.MethodPerEndpoint]( + this.injector, + st.HomeMixer.ExecutePipeline + ) + ).add[HomeHttpController] + + override def configureHttpsServer(server: Http.Server): Http.Server = { + val serviceIdentifier: ServiceIdentifier = injector.instance[ServiceIdentifier] + server.withMutualTls(serviceIdentifier.copy(role = "home-mixer")) + } override val dest: String = "/s/home-mixer/home-mixer:strato" diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala index 982b77487..ac79db57d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala @@ -3,7 +3,6 @@ package com.twitter.home_mixer import com.twitter.finagle.thrift.ClientId import com.twitter.finatra.thrift.routing.ThriftWarmup import com.twitter.home_mixer.{thriftscala => st} -import com.twitter.util.logging.Logging import com.twitter.inject.utils.Handler import com.twitter.product_mixer.core.{thriftscala => pt} import com.twitter.scrooge.Request @@ -11,6 +10,7 @@ import com.twitter.scrooge.Response import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Try +import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton @@ -19,45 +19,73 @@ class HomeMixerThriftServerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler with Logging { - private val clientId = ClientId("thrift-warmup-client") + private val CurrentClientId = ClientId("thrift-warmup-client") + private val SleepThreshold = 10000 // millis + + private val TestIds = Seq.empty + + private val BaseClientContext = pt.ClientContext( + userId = TestIds.headOption, + guestId = None, + appId = Some(1L), + ipAddress = Some("0.0.0.0"), + userAgent = Some("FAKE_USER_AGENT_FOR_WARMUPS"), + countryCode = Some("US"), + languageCode = Some("en"), + isTwoffice = None, + userRoles = None, + deviceId = Some("FAKE_DEVICE_ID_FOR_WARMUPS") + ) def handle(): Unit = { - val testIds = Seq(1, 2, 3) try { - clientId.asCurrent { - testIds.foreach { id => - val warmupReq = warmupQuery(id) - info(s"Sending warm-up request to service with query: $warmupReq") - warmup.sendRequest( - method = st.HomeMixer.GetUrtResponse, - req = Request(st.HomeMixer.GetUrtResponse.Args(warmupReq)))(assertWarmupResponse) + CurrentClientId.asCurrent { + TestIds.foreach { id => + val warmupReqs = warmupQuery(id) + warmupReqs.foreach { warmupReq => + info(s"Sending warm-up request to service with query: $warmupReq") + warmup.sendRequest( + method = st.HomeMixer.GetUrtResponse, + req = Request(st.HomeMixer.GetUrtResponse.Args(warmupReq)), + )(assertWarmupResponse) + } } } } catch { - case e: Throwable => error(e.getMessage, e) + case e: Throwable => + error(e.getMessage, e) } info("Warm-up done.") } - private def warmupQuery(userId: Long): st.HomeMixerRequest = { - val clientContext = pt.ClientContext( - userId = Some(userId), - guestId = None, - appId = Some(12345L), - ipAddress = Some("0.0.0.0"), - userAgent = Some("FAKE_USER_AGENT_FOR_WARMUPS"), - countryCode = Some("US"), - languageCode = Some("en"), - isTwoffice = None, - userRoles = None, - deviceId = Some("FAKE_DEVICE_ID_FOR_WARMUPS") + private def warmupQuery(userId: Long): Seq[st.HomeMixerRequest] = { + val clientContext = BaseClientContext.copy(userId = Some(userId)) + + val scoredTweets = st.HomeMixerRequest( + clientContext = clientContext, + product = st.Product.ScoredTweets, + productContext = Some(st.ProductContext.ScoredTweets(st.ScoredTweets())), + ) + + val scoredVideoTweets = st.HomeMixerRequest( + clientContext = clientContext, + product = st.Product.ScoredVideoTweets, + productContext = Some(st.ProductContext.ScoredVideoTweets(st.ScoredVideoTweets())), ) - st.HomeMixerRequest( + + val forYou = st.HomeMixerRequest( + clientContext = clientContext, + product = st.Product.ForYou, + productContext = Some(st.ProductContext.ForYou(st.ForYou())), + ) + + val following = st.HomeMixerRequest( clientContext = clientContext, product = st.Product.Following, productContext = Some(st.ProductContext.Following(st.Following())), - maxResults = Some(3) ) + + Seq(scoredTweets, scoredVideoTweets, forYou, following) } private def assertWarmupResponse( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel index 3f738bcb6..daf9660ec 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel @@ -4,21 +4,23 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/stale_tweets", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/transformer/stale_tweets", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala index d26843193..9c603dcf6 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala @@ -2,20 +2,26 @@ package com.twitter.home_mixer.candidate_pipeline import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetTypeMetricsFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter +import com.twitter.home_mixer.functional_component.filter.LocationFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter +import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.communities.CommunityNamesFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator import com.twitter.product_mixer.component_library.filter.FeatureFilter import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate @@ -39,9 +45,12 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( conversationServiceCandidateSource: ConversationServiceCandidateSource, tweetypieFeatureHydrator: TweetypieFeatureHydrator, namesFeatureHydrator: NamesFeatureHydrator, + communityNamesFeatureHydrator: CommunityNamesFeatureHydrator, invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, override val gates: Seq[BaseGate[Query]], - override val decorator: Option[CandidateDecorator[Query, TweetCandidate]]) + override val decorator: Option[CandidateDecorator[Query, TweetCandidate]], + servedType: hmt.ServedType, + paramGatedPostContextFeatureHydrator: ParamGatedBulkCandidateFeatureHydrator[Query, TweetCandidate]) extends DependentCandidatePipelineConfig[ Query, ConversationServiceCandidateSourceRequest, @@ -52,7 +61,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ConversationService") - private val TweetypieHydratedFilterId = "TweetypieHydrated" + private val InNetworkFilterId = "InNetwork" private val QuotedTweetDroppedFilterId = "QuotedTweetDropped" override val candidateSource: BaseCandidateSource[ @@ -64,23 +73,24 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( Query, ConversationServiceCandidateSourceRequest ] = { (_, candidates) => - val tweetsWithConversationMetadata = candidates.map { candidate => - TweetWithConversationMetadata( - tweetId = candidate.candidateIdLong, - userId = candidate.features.getOrElse(AuthorIdFeature, None), - sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), - sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None), - inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), - conversationId = None, - ancestors = Seq.empty - ) + val tweetsWithConversationMetadata = candidates.collect { + case candidate if candidate.isCandidateType[TweetCandidate]() => + TweetWithConversationMetadata( + tweetId = candidate.candidateIdLong, + userId = candidate.features.getOrElse(AuthorIdFeature, None), + sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), + sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None), + inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), + conversationId = None, + ancestors = Seq.empty + ) } ConversationServiceCandidateSourceRequest(tweetsWithConversationMetadata) } override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[TweetWithConversationMetadata] - ] = Seq(ConversationServiceResponseFeatureTransformer) + ] = Seq(ConversationServiceResponseFeatureTransformer(servedType)) override val resultTransformer: CandidatePipelineResultsTransformer[ TweetWithConversationMetadata, @@ -96,18 +106,25 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( override def filters: Seq[Filter[Query, TweetCandidate]] = Seq( RetweetDeduplicationFilter, - FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), + FeatureFilter.fromFeature(FilterIdentifier(InNetworkFilterId), InNetworkFeature), + TweetHydrationFilter, PredicateFeatureFilter.fromPredicate( FilterIdentifier(QuotedTweetDroppedFilterId), shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } ), + LocationFilter, invalidSubscriptionTweetFilter, InvalidConversationModuleFilter ) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[Query, TweetCandidate, _] - ] = Seq(namesFeatureHydrator) + ] = Seq( + communityNamesFeatureHydrator, + namesFeatureHydrator, + TweetTypeMetricsFeatureHydrator, + paramGatedPostContextFeatureHydrator + ) override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala index bb55f85e3..6776326cb 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala @@ -1,12 +1,19 @@ package com.twitter.home_mixer.candidate_pipeline +import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetContextBuilder +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.PostContextFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.gate.BaseGate +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.communities.CommunityNamesFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator +import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.timelines.configapi.Param import com.twitter.product_mixer.core.pipeline.PipelineQuery import javax.inject.Inject import javax.inject.Singleton @@ -15,20 +22,37 @@ import javax.inject.Singleton class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() ( conversationServiceCandidateSource: ConversationServiceCandidateSource, tweetypieFeatureHydrator: TweetypieFeatureHydrator, + communityNamesFeatureHydrator: CommunityNamesFeatureHydrator, + postContextFeatureHydrator: PostContextFeatureHydrator, invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - namesFeatureHydrator: NamesFeatureHydrator) { + namesFeatureHydrator: NamesFeatureHydrator, + homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, + homeTweetContextBuilder: HomeTweetContextBuilder) { def build( - gates: Seq[BaseGate[Query]] = Seq.empty, - decorator: Option[CandidateDecorator[Query, TweetCandidate]] = None + nonEmptyCandidateScope: CandidateScope, + servedType: hmt.ServedType, + enablePostContextFeatureHydratorParam: Param[Boolean] ): ConversationServiceCandidatePipelineConfig[Query] = { + val paramGatedPostContextFeatureHydrator = ParamGatedBulkCandidateFeatureHydrator( + enablePostContextFeatureHydratorParam, + postContextFeatureHydrator + ) + new ConversationServiceCandidatePipelineConfig( conversationServiceCandidateSource, tweetypieFeatureHydrator, namesFeatureHydrator, + communityNamesFeatureHydrator, invalidSubscriptionTweetFilter, - gates, - decorator + Seq(NonEmptyCandidatesGate(nonEmptyCandidateScope)), + HomeConversationServiceCandidateDecorator( + homeFeedbackActionInfoBuilder, + homeTweetContextBuilder, + servedType + ), + servedType, + paramGatedPostContextFeatureHydrator ) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.scala index 154c080ad..b819c7225 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.scala @@ -1,15 +1,15 @@ package com.twitter.home_mixer.candidate_pipeline import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -object ConversationServiceResponseFeatureTransformer +case class ConversationServiceResponseFeatureTransformer(servedType: hmt.ServedType) extends CandidateFeatureTransformer[TweetWithConversationMetadata] { override val identifier: TransformerIdentifier = @@ -23,7 +23,7 @@ object ConversationServiceResponseFeatureTransformer SourceUserIdFeature, ConversationModuleFocalTweetIdFeature, AncestorsFeature, - SuggestTypeFeature + ServedTypeFeature ) override def transform(candidate: TweetWithConversationMetadata): FeatureMap = FeatureMapBuilder() @@ -34,6 +34,6 @@ object ConversationServiceResponseFeatureTransformer .add(SourceUserIdFeature, candidate.sourceUserId) .add(ConversationModuleFocalTweetIdFeature, candidate.conversationId) .add(AncestorsFeature, candidate.ancestors) - .add(SuggestTypeFeature, Some(SuggestType.RankedOrganicTweet)) + .add(ServedTypeFeature, servedType) .build() } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala index d9bb73695..9948bc972 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala @@ -1,15 +1,16 @@ package com.twitter.home_mixer.candidate_pipeline -import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer +import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.product_mixer.component_library.candidate_source.stale_tweets.StaleTweetsCacheCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.EmptyClientEventInfoBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.transformer.stale_tweets.EditedTweetsCandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator @@ -44,15 +45,12 @@ case class EditedTweetsCandidatePipelineConfig @Inject() ( override val candidateSource: BaseCandidateSource[Seq[Long], Long] = staleTweetsCacheCandidateSource - override val queryTransformer: CandidatePipelineQueryTransformer[ - PipelineQuery, - Seq[Long] - ] = EditedTweetsCandidatePipelineQueryTransformer + override val queryTransformer: CandidatePipelineQueryTransformer[PipelineQuery, Seq[Long]] = + EditedTweetsCandidatePipelineQueryTransformer(PersistenceEntriesFeature) - override val resultTransformer: CandidatePipelineResultsTransformer[ - Long, - TweetCandidate - ] = { candidate => TweetCandidate(id = candidate) } + override val resultTransformer: CandidatePipelineResultsTransformer[Long, TweetCandidate] = { + candidate => TweetCandidate(id = candidate) + } override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] @@ -78,7 +76,6 @@ case class EditedTweetsCandidatePipelineConfig @Inject() ( Some(UrtItemCandidateDecorator(tweetItemBuilder)) } - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.5, 50, 60, 60) - ) + override val alerts = + Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.5, 50, 60, 60)) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/VerifiedPromptCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/VerifiedPromptCandidatePipelineConfig.scala new file mode 100644 index 000000000..613dc4f94 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/VerifiedPromptCandidatePipelineConfig.scala @@ -0,0 +1,75 @@ +package com.twitter.home_mixer.candidate_pipeline + +import com.twitter.home_mixer.functional_component.decorator.builder.VerifiedPromptBuilder +import com.twitter.home_mixer.functional_component.gate.RateLimitNotGate +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.model.candidate.InlinePromptCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.candidate_source.IdentityCandidateExtractor +import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.ExternalStringRegistry +import com.twitter.stringcenter.client.StringCenter +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +class VerifiedPromptCandidatePipelineConfig @Inject() ( + identityCandidateExtractor: IdentityCandidateExtractor[PipelineQuery], + @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry], + @ProductScoped stringCenterProvider: Provider[StringCenter]) + extends DependentCandidatePipelineConfig[ + PipelineQuery, + PipelineQuery, + PipelineQuery, + InlinePromptCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("VerifiedPrompt") + + override val gates: Seq[Gate[PipelineQuery]] = Seq(RateLimitNotGate) + + override def candidateSource: CandidateSource[ + PipelineQuery, + PipelineQuery + ] = PassthroughCandidateSource( + CandidateSourceIdentifier("VerifiedPassthroughCandidateSource"), + identityCandidateExtractor + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[PipelineQuery, PipelineQuery] = + identity + + override val resultTransformer: CandidatePipelineResultsTransformer[ + PipelineQuery, + InlinePromptCandidate + ] = { query => InlinePromptCandidate(id = query.getRequiredUserId.toString) } + + override val decorator: Option[ + CandidateDecorator[PipelineQuery, InlinePromptCandidate] + ] = { + val clientEventInfoBuilder = + ClientEventInfoBuilder[PipelineQuery, InlinePromptCandidate]("verified_prompt") + val stringCenter = stringCenterProvider.get() + val externalStringRegistry = externalStringRegistryProvider.get() + + val verifiedPromptBuilder = VerifiedPromptBuilder( + clientEventInfoBuilder, + stringCenter, + externalStringRegistry + ) + + Some(UrtItemCandidateDecorator(verifiedPromptBuilder)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel index dfaff319d..c893140b5 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel @@ -4,6 +4,7 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ + "finatra/http-server/src/main/scala/com/twitter/finatra/http", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeHttpController.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeHttpController.scala new file mode 100644 index 000000000..20e70979c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeHttpController.scala @@ -0,0 +1,94 @@ +package com.twitter.home_mixer.controller + +import com.twitter.finagle.http.Request +import com.twitter.finatra.http.Controller +import com.twitter.home_mixer.model.request.HomeMixerRequest +import com.twitter.home_mixer.model.request.ScoredTweetsProduct +import com.twitter.home_mixer.model.request.ScoredTweetsProductContext +import com.twitter.home_mixer.service.ScoredTweetsService +import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.stitch.Stitch +import com.twitter.timelines.configapi.Params +import com.twitter.util.jackson.ScalaObjectMapper +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HomeHttpController @Inject() ( + scoredTweetsService: ScoredTweetsService, + paramsBuilder: ParamsBuilder, + mapper: ScalaObjectMapper) + extends Controller { + + private val UserIdParam = "userId" + private val CountryCode = "US" + private val LanguageCode = "en" + private val AppId = 1L + private val DefaultUserId = 1L + private val UserAgent = "" + + private val BaseClientContext = ClientContext( + userId = Some(DefaultUserId), + guestId = None, + appId = Some(AppId), + ipAddress = None, + userAgent = Some(UserAgent), + countryCode = Some(CountryCode), + languageCode = Some(LanguageCode), + isTwoffice = None, + userRoles = None, + deviceId = None, + mobileDeviceId = None, + mobileDeviceAdId = None, + limitAdTracking = None, + guestIdAds = None, + guestIdMarketing = None, + authenticatedUserId = None, + isVerifiedCrawler = None + ) + + case class ScoredTweetMetadata(tweetId: Long, authorId: Long, inNetwork: Boolean, text: String) + + get("/scoredTweets") { request: Request => + val userId = request.getLongParam(UserIdParam) + val hmRequest = HomeMixerRequest( + clientContext = BaseClientContext.copy(userId = Some(userId)), + product = ScoredTweetsProduct, + productContext = + Some(ScoredTweetsProductContext(None, None, None, None, None, None, None, None)), + serializedRequestCursor = None, + maxResults = None, + debugParams = None, + homeRequestParam = false + ) + + val params = buildParams(hmRequest) + val response = scoredTweetsService.getScoredTweetsResponse[HomeMixerRequest](hmRequest, params) + Stitch.run(response).map { scoredTweetsResponse => + scoredTweetsResponse.scoredTweets.map { tweet => + val tweetData = ScoredTweetMetadata( + tweet.tweetId, + tweet.authorId, + tweet.inNetwork.getOrElse(false), + tweet.tweetText.getOrElse("") + ) + mapper.writeValueAsString(tweetData) + } + } + } + + private def buildParams(request: HomeMixerRequest): Params = { + val userAgeOpt = request.clientContext.userId.map { userId => + SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) + } + val fsCustomMapInput = userAgeOpt.map("account_age_in_days" -> _).toMap + paramsBuilder.build( + clientContext = request.clientContext, + product = request.product, + featureOverrides = request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), + fsCustomMapInput = fsCustomMapInput + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala index fc1b7770e..f068c6866 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala @@ -13,11 +13,14 @@ import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Params import javax.inject.Inject +import server.src.main.scala.com.twitter.home_mixer.service.HeavyRankerScoresService class HomeThriftController @Inject() ( homeRequestUnmarshaller: HomeMixerRequestUnmarshaller, + debugQueryService: DebugQueryService, urtService: UrtService, scoredTweetsService: ScoredTweetsService, + heavyRankerScoresService: HeavyRankerScoresService, paramsBuilder: ParamsBuilder) extends Controller(t.HomeMixer) with DebugTwitterContext { @@ -28,6 +31,18 @@ class HomeThriftController @Inject() ( Stitch.run(urtService.getUrtResponse[HomeMixerRequest](request, params)) } + handle(t.HomeMixer.DebugGetUrtResponse) { args: t.HomeMixer.DebugGetUrtResponse.Args => + val request = homeRequestUnmarshaller(args.request) + val params = buildParams(request) + withDebugTwitterContext(request.clientContext) { + Stitch.run(urtService.getUrtResponse[HomeMixerRequest](request, params)) + } + } + + // Handle debug requests + handle(t.HomeMixer.ExecutePipeline) + .withService(debugQueryService(homeRequestUnmarshaller.apply)) + handle(t.HomeMixer.GetScoredTweetsResponse) { args: t.HomeMixer.GetScoredTweetsResponse.Args => val request = homeRequestUnmarshaller(args.request) val params = buildParams(request) @@ -36,8 +51,19 @@ class HomeThriftController @Inject() ( } } + handle(t.HomeMixer.GetHeavyRankerScoresResponse) { + args: t.HomeMixer.GetHeavyRankerScoresResponse.Args => + val request = homeRequestUnmarshaller(args.request) + val params = buildParams(request) + withDebugTwitterContext(request.clientContext) { + Stitch.run( + heavyRankerScoresService.getHeavyRankerScoresResponse[HomeMixerRequest](request, params)) + } + } + private def buildParams(request: HomeMixerRequest): Params = { val userAgeOpt = request.clientContext.userId.map { userId => + // Setting to Int.MaxValue for cases where id is not snowflake id as they are pretty old accounts SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) } val fsCustomMapInput = userAgeOpt.map("account_age_in_days" -> _).toMap diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel index 30ee81acc..3a5b105c3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel @@ -7,18 +7,12 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/thrift/src/main/thrift:thrift-scala", + "joinkey/src/main/scala/com/twitter/joinkey/context", + "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "stitch/stitch-repo/src/main/scala", + "stitch/stitch-gizmoduck", "strato/config/columns/auth-context:auth-context-strato-client", - "strato/config/columns/gizmoduck:gizmoduck-strato-client", - "strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala", "strato/src/main/scala/com/twitter/strato/callcontext", - "strato/src/main/scala/com/twitter/strato/fed", "strato/src/main/scala/com/twitter/strato/fed/server", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala index 0ef27b7a0..938d85a3c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala @@ -1,9 +1,17 @@ package com.twitter.home_mixer.federated +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.filter.SLODefinition +import com.twitter.finagle.filter.SLOStatsFilter +import com.twitter.finagle.service.ResponseClassifier +import com.twitter.finagle.stats.DefaultStatsReceiver import com.twitter.gizmoduck.{thriftscala => gd} import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.{thriftscala => hm} +import com.twitter.joinkey.context.thriftscala.{RequestJoinKeyContext => ThriftJoinKeyContext} +import com.twitter.joinkey.context.RequestJoinKeyContext +import com.twitter.joinkey.context.ThreadedLocalJoinKeyRandomGenerator import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult @@ -11,15 +19,17 @@ import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry import com.twitter.product_mixer.core.{thriftscala => pm} import com.twitter.stitch.Arrow import com.twitter.stitch.Stitch +import com.twitter.stitch.Stitch.Letter +import com.twitter.stitch.gizmoduck.Gizmoduck import com.twitter.strato.callcontext.CallContext import com.twitter.strato.catalog.OpMetadata import com.twitter.strato.config._ import com.twitter.strato.data._ import com.twitter.strato.fed.StratoFed import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn -import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn import com.twitter.strato.graphql.timelines.{thriftscala => gql} import com.twitter.strato.thrift.ScroogeConv +import com.twitter.timelines.render.thriftscala.Timeline import com.twitter.timelines.render.{thriftscala => tr} import com.twitter.util.Try import javax.inject.Inject @@ -28,7 +38,7 @@ import javax.inject.Singleton @Singleton class HomeMixerColumn @Inject() ( homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller, - compositeOnUserClientColumn: CompositeOnUserClientColumn, + gizmoduck: Gizmoduck, auditIpClientColumn: AuditIpClientColumn, paramsBuilder: ParamsBuilder, productPipelineRegistry: ProductPipelineRegistry) @@ -38,6 +48,7 @@ class HomeMixerColumn @Inject() ( override val contactInfo: ContactInfo = ContactInfo( contactEmail = "", ldapGroup = "", + jiraProject = "", slackRoomId = "" ) @@ -57,6 +68,18 @@ class HomeMixerColumn @Inject() ( zone = Seq("")) ) + private val sloStatsFilter = { + val requestToSLODefinition: PartialFunction[Any, SLODefinition] = { + case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) => + SLODefinition("ForYou", 1000.millis) + } + new SLOStatsFilter[gql.TimelineKey, Result[Timeline]]( + requestToSLODefinition = requestToSLODefinition, + responseClassifier = ResponseClassifier.Default, + statsReceiver = DefaultStatsReceiver.scope("slo") + ) + } + override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers) override type Key = gql.TimelineKey @@ -67,25 +90,30 @@ class HomeMixerColumn @Inject() ( override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView] override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline] + // For populating requestJoinId + private val joinIdGenerator = new ThreadedLocalJoinKeyRandomGenerator + // For populating user roles + private val queryFieldsRoles = gd.QueryFields.Roles + private object RequestJoinKeyLetter extends Letter[Option[Long]] { + override def let[A](requestJoinId: Option[Long])(fn: => A): A = + RequestJoinKeyContext.let(ThriftJoinKeyContext(requestJoinId))(fn) + } + private def createHomeMixerRequestArrow( - compositeOnUserClientColumn: CompositeOnUserClientColumn, - auditIpClientColumn: AuditIpClientColumn + auditIpClientColumn: AuditIpClientColumn, ): Arrow[(Key, View), hm.HomeMixerRequest] = { val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = { - val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) = - (gd.LookupContext(), Set(gd.QueryFields.Roles)) - val populateUserRoles = Arrow .flatMap[(Key, View), Option[Set[String]]] { _ => Stitch.collect { CallContext.twitterUserId.map { userId => - compositeOnUserClientColumn.fetcher - .callStack(HomeMixerColumn.FetchCallstack) - .fetch(userId, gizmoduckView).map(_.v) - .map { - _.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty) - } + gizmoduck + .getUserById( + userId = userId, + queryFields = Set(queryFieldsRoles), + context = gd.LookupContext(forUserId = Some(userId)) + ).map(_.roles.map(_.roles.toSet).getOrElse(Set.empty)) } } } @@ -152,7 +180,9 @@ class HomeMixerColumn @Inject() ( deviceId = CallContext.deviceId, mobileDeviceId = CallContext.mobileDeviceId, mobileDeviceAdId = CallContext.adId, - limitAdTracking = CallContext.limitAdTracking + limitAdTracking = CallContext.limitAdTracking, + authenticatedUserId = CallContext.authenticatedTwitterUserId, + isVerifiedCrawler = CallContext.isVerifiedCrawler ) hm.HomeMixerRequest( @@ -173,18 +203,16 @@ class HomeMixerColumn @Inject() ( Arrow .identity[(Key, View)] .andThen { - createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn) + createHomeMixerRequestArrow(auditIpClientColumn) } - .map { - case thriftRequest => - val request = homeMixerRequestUnmarshaller(thriftRequest) - val params = paramsBuilder.build( - clientContext = request.clientContext, - product = request.product, - featureOverrides = - request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), - ) - ProductPipelineRequest(request, params) + .map { thriftRequest => + val request = homeMixerRequestUnmarshaller(thriftRequest) + val params = paramsBuilder.build( + clientContext = request.clientContext, + product = request.product, + featureOverrides = request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), + ) + ProductPipelineRequest(request, params) } } @@ -194,19 +222,33 @@ class HomeMixerColumn @Inject() ( ] = Arrow .identity[ProductPipelineRequest[HomeMixerRequest]] .map { pipelineRequest => - val pipelineArrow = productPipelineRegistry - .getProductPipeline[HomeMixerRequest, tr.TimelineResponse]( - pipelineRequest.request.product) - .arrow + val pipelineArrow = + Arrow.let(RequestJoinKeyLetter)(joinIdGenerator.get().getNewRequestJoinId) { // Populate requestJoinId + productPipelineRegistry + .getProductPipeline[HomeMixerRequest, tr.TimelineResponse]( + pipelineRequest.request.product) + .arrow + } (pipelineArrow, pipelineRequest) }.applyArrow - transformThriftIntoPipelineRequest.andThen(underlyingProduct).map { - _.result match { - case Some(result) => found(result.timeline) - case _ => missing - } - } + Arrow + .zipWithArg( + Arrow.time { + transformThriftIntoPipelineRequest + .andThen(underlyingProduct) + .map { + _.result match { + case Some(result) => found(result.timeline) + case _ => missing + } + } + } + ).map { + case ((key, _), (result, duration)) => + sloStatsFilter.record(key, result, duration.inNanoseconds) + result + }.lowerFromTry } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel index 4df69c9f9..4c81e6e85 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel @@ -4,25 +4,14 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "finagle/finagle-core/src/main", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/pivot", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/scala/com/twitter/suggests/controller_data", - "src/thrift/com/twitter/suggests/controller_data:controller_data-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", - "stringcenter/client", - "stringcenter/client/src/main/java", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/trends_events", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/EntryPointPivotModuleDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/EntryPointPivotModuleDecorator.scala new file mode 100644 index 000000000..ceaa832ba --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/EntryPointPivotModuleDecorator.scala @@ -0,0 +1,62 @@ +package com.twitter.home_mixer.functional_component.decorator + +import com.twitter.home_mixer.functional_component.decorator.EntryPointPivotModuleCandidateDecorator.EntryNamespaceString +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.pivot.PivotCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.SuggestTypeClientEventDetailsBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleHeaderBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder +import com.twitter.product_mixer.component_library.model.candidate.pivot.PivotCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr +import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Vertical +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelineservice.suggests.{thriftscala => st} + +case class StrEntryPointPivotCategoryText( + defaultText: String) + extends BaseStr[PipelineQuery, PivotCandidate] { + def apply( + query: PipelineQuery, + candidate: PivotCandidate, + candidateFeatures: FeatureMap + ): String = + candidate.categoryText.getOrElse(defaultText) +} +object EntryPointPivotModuleCandidateDecorator { + val EntryNamespaceString = "entry-point-pivot" +} +case class EntryPointPivotModuleCandidateDecorator( + component: String, + headerText: BaseStr[PipelineQuery, PivotCandidate]) { + + private val clientEventsBuilder = + ClientEventInfoBuilder[PipelineQuery, PivotCandidate](component) + private val clientEventDetailsBuilder = + SuggestTypeClientEventDetailsBuilder(st.SuggestType.EntryPointPivot) + + private val itemBuilder = + PivotCandidateUrtItemBuilder(clientEventInfoBuilder = Some(clientEventDetailsBuilder)) + private val itemDecorator = UrtItemCandidateDecorator(itemBuilder) + + private val moduleHeaderBuilder = ModuleHeaderBuilder( + textBuilder = headerText, + isSticky = Some(false), + urlBuilder = None + ) + + private val moduleBuilder = TimelineModuleBuilder( + entryNamespace = EntryNamespace(EntryNamespaceString), + displayTypeBuilder = StaticModuleDisplayTypeBuilder(Vertical), + clientEventInfoBuilder = clientEventsBuilder, + headerBuilder = Some(moduleHeaderBuilder), + footerBuilder = None + ) + + val moduleDecorator: UrtItemInModuleDecorator[PipelineQuery, PivotCandidate, Nothing] = + UrtItemInModuleDecorator(itemDecorator, moduleBuilder) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ForYouTweetCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ForYouTweetCandidateDecorator.scala new file mode 100644 index 000000000..e7443ade2 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ForYouTweetCandidateDecorator.scala @@ -0,0 +1,64 @@ +package com.twitter.home_mixer.functional_component.decorator + +import com.twitter.home_mixer.functional_component.decorator.ForYouTweetCandidateDecorator.ConvoModuleEntryNamespace +import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder +import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetContextBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetSocialContextBuilder +import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ManualModuleId +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +import javax.inject.Inject +import javax.inject.Singleton + +object ForYouTweetCandidateDecorator { + val ConvoModuleEntryNamespace = EntryNamespace("home-conversation") +} + +@Singleton +class ForYouTweetCandidateDecorator @Inject() ( + homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, + homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder, + homeTweetContextBuilder: HomeTweetContextBuilder) { + + val clientEventInfoBuilder = HomeClientEventInfoBuilder() + + val tweetItemBuilder = TweetCandidateUrtItemBuilder( + clientEventInfoBuilder = clientEventInfoBuilder, + socialContextBuilder = Some(homeTweetSocialContextBuilder), + timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), + feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder), + tweetContext = Some(homeTweetContextBuilder) + ) + + val tweetDecorator = UrtItemCandidateDecorator(tweetItemBuilder) + + val moduleBuilder = TimelineModuleBuilder( + entryNamespace = ConvoModuleEntryNamespace, + clientEventInfoBuilder = clientEventInfoBuilder, + moduleIdGeneration = ManualModuleId(0L), + displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), + metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) + ) + + val decorator: UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long] = + UrtMultipleModulesDecorator( + urtItemCandidateDecorator = tweetDecorator, + moduleBuilder = moduleBuilder, + groupByKey = (_, _, candidateFeatures) => + candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) + ) + + def build(): UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long] = decorator +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala index d145391d4..ee9a1293c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala @@ -1,36 +1,39 @@ package com.twitter.home_mixer.functional_component.decorator +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.{HomeFeedbackActionInfoBuilder, HomeTweetContextBuilder} import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.social_context.CommunitiesSocialContextBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} object HomeConversationServiceCandidateDecorator { private val ConversationModuleNamespace = EntryNamespace("home-conversation") def apply( - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder + homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, + homeTweetContextBuilder: HomeTweetContextBuilder, + servedType: hmt.ServedType ): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = { - val suggestType = st.SuggestType.RankedOrganicTweet - val component = InjectionScribeUtil.scribeComponent(suggestType).get + val component = servedType.originalName val clientEventInfoBuilder = ClientEventInfoBuilder(component) val tweetItemBuilder = TweetCandidateUrtItemBuilder( clientEventInfoBuilder = clientEventInfoBuilder, timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) + feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder), + socialContextBuilder = Some(CommunitiesSocialContextBuilder), + tweetContext = Some(homeTweetContextBuilder) ) val moduleBuilder = TimelineModuleBuilder( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/KeywordTrendsModuleCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/KeywordTrendsModuleCandidateDecorator.scala new file mode 100644 index 000000000..af2b96aa4 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/KeywordTrendsModuleCandidateDecorator.scala @@ -0,0 +1,68 @@ +package com.twitter.home_mixer.functional_component.decorator + +import com.twitter.home_mixer.functional_component.decorator.KeywordTrendsModuleCandidateDecorator.Component +import com.twitter.home_mixer.functional_component.decorator.KeywordTrendsModuleCandidateDecorator.KeywordTrendsEntryNamespace +import com.twitter.home_mixer.functional_component.decorator.KeywordTrendsModuleCandidateDecorator.TrendsUrl +import com.twitter.home_mixer.functional_component.decorator.builder.KeywordTrendMetaDescriptionBuilder +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.TrendCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.TrendPromotedMetadataBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.TrendRankBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.StaticUrlBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleHeaderBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder +import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate +import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ExternalUrl +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Vertical +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import javax.inject.Inject +import javax.inject.Singleton + +object KeywordTrendsModuleCandidateDecorator { + val KeywordTrendsEntryNamespace = EntryNamespace("keyword-trends") + private val Component = "trend" + private val TrendsUrl = "" +} + +@Singleton +class KeywordTrendsModuleCandidateDecorator @Inject() ( + keywordTrendMetaDescriptionBuilder: KeywordTrendMetaDescriptionBuilder, + @ProductScoped stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) { + + private val clientEventInfoBuilder = + ClientEventInfoBuilder[PipelineQuery, UnifiedTrendCandidate](Component) + + private val trendItemBuilder = TrendCandidateUrtItemBuilder( + keywordTrendMetaDescriptionBuilder, + TrendPromotedMetadataBuilder, + clientEventInfoBuilder, + Some(TrendRankBuilder[PipelineQuery, UnifiedTrendCandidate]()) + ) + private val trendItemDecorator = UrtItemCandidateDecorator(trendItemBuilder) + + private val moduleHeaderBuilder = ModuleHeaderBuilder( + textBuilder = Str( + text = externalStrings.TrendingString, + stringCenter = stringCenter + ), + isSticky = Some(false), + urlBuilder = Some(StaticUrlBuilder(TrendsUrl, ExternalUrl)) + ) + private val moduleBuilder = TimelineModuleBuilder( + entryNamespace = KeywordTrendsEntryNamespace, + displayTypeBuilder = StaticModuleDisplayTypeBuilder(Vertical), + clientEventInfoBuilder = clientEventInfoBuilder, + headerBuilder = Some(moduleHeaderBuilder) + ) + + val moduleDecorator = UrtItemInModuleDecorator(trendItemDecorator, moduleBuilder) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/PinnedTweetBroadcastCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/PinnedTweetBroadcastCandidateDecorator.scala new file mode 100644 index 000000000..993d7429c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/PinnedTweetBroadcastCandidateDecorator.scala @@ -0,0 +1,71 @@ +package com.twitter.home_mixer.functional_component.decorator + +import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.PinnedTweetsModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselType +import com.twitter.home_mixer.param.HomeGlobalParams.EnableLandingPage +import com.twitter.home_mixer.param.HomeGlobalParams.EnablePinnedTweetsCarouselParam +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.social_context.GeneralSocialContextBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.decorator.Decoration +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ExternalUrl +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.PinGeneralContextType +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stitch.Stitch +import com.twitter.stringcenter.client.StringCenter +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +class PinnedTweetBroadcastCandidateDecorator @Inject() ( + tweetCarouselModuleCandidateDecorator: TweetCarouselModuleCandidateDecorator, + @ProductScoped stringCenterProvider: Provider[StringCenter], + externalStrings: HomeMixerExternalStrings, + homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) + extends CandidateDecorator[PipelineQuery, TweetCandidate] { + + private val stringCenter = stringCenterProvider.get() + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[Decoration]] = { + if (candidates.size > 1 && query.params(EnablePinnedTweetsCarouselParam)) { + tweetCarouselModuleCandidateDecorator + .build(TweetCarouselType.PinnedTweets).apply(query, candidates) + } else { + val clientEventInfoBuilder = HomeClientEventInfoBuilder[PipelineQuery, TweetCandidate]() + + val landingUrl = if (query.params(EnableLandingPage)) { + PinnedTweetsModuleCandidateDecorator.headerLink.map(link => Url(ExternalUrl, link)) + } else None + + val socialContextBuilder = GeneralSocialContextBuilder( + textBuilder = Str(externalStrings.BroadcastedPinnedTweetSocialContextString, stringCenter), + contextType = PinGeneralContextType, + landingUrl = landingUrl + ) + + val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( + clientEventInfoBuilder = clientEventInfoBuilder, + socialContextBuilder = Some(socialContextBuilder), + timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), + feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) + ) + + UrtItemCandidateDecorator(tweetItemBuilder).apply(query, candidates) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/StoriesModuleCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/StoriesModuleCandidateDecorator.scala new file mode 100644 index 000000000..ac7ef9a1e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/StoriesModuleCandidateDecorator.scala @@ -0,0 +1,70 @@ +package com.twitter.home_mixer.functional_component.decorator + +import com.twitter.home_mixer.functional_component.decorator.StoriesModuleCandidateDecorator.Component +import com.twitter.home_mixer.functional_component.decorator.StoriesModuleCandidateDecorator.TrendsEntryNamespace +import com.twitter.home_mixer.functional_component.decorator.StoriesModuleCandidateDecorator.TrendsUrl +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.AiTrendCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.StaticUrlBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleFooterBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleHeaderBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder +import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate +import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DeepLink +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Vertical +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import javax.inject.Inject +import javax.inject.Singleton + +object StoriesModuleCandidateDecorator { + val TrendsEntryNamespace = EntryNamespace("stories") + private val Component = "stories" + private val TrendsUrl = "" +} + +@Singleton +class StoriesModuleCandidateDecorator @Inject() ( + @ProductScoped stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) { + private val clientEventsBuilder = + ClientEventInfoBuilder[PipelineQuery, UnifiedTrendCandidate](Component) + + private val trendItemBuilder = AiTrendCandidateUrtItemBuilder(clientEventsBuilder) + private val trendItemDecorator = UrtItemCandidateDecorator(trendItemBuilder) + + private val moduleHeaderBuilder = ModuleHeaderBuilder( + textBuilder = Str( + text = externalStrings.NewsHeaderString, + stringCenter = stringCenter + ), + isSticky = Some(false), + urlBuilder = Some(StaticUrlBuilder(TrendsUrl, DeepLink)) + ) + + private val moduleFooterBuilder = ModuleFooterBuilder( + textBuilder = Str( + text = externalStrings.NewsFooterString, + stringCenter = stringCenter + ), + urlBuilder = Some(StaticUrlBuilder(TrendsUrl, DeepLink)) + ) + + private val moduleBuilder = TimelineModuleBuilder( + entryNamespace = TrendsEntryNamespace, + displayTypeBuilder = StaticModuleDisplayTypeBuilder(Vertical), + clientEventInfoBuilder = clientEventsBuilder, + headerBuilder = Some(moduleHeaderBuilder), + footerBuilder = Some(moduleFooterBuilder) + ) + + val moduleDecorator: UrtItemInModuleDecorator[PipelineQuery, UnifiedTrendCandidate, Nothing] = + UrtItemInModuleDecorator(trendItemDecorator, moduleBuilder) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TuneFeedModuleCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TuneFeedModuleCandidateDecorator.scala new file mode 100644 index 000000000..fd79740f4 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TuneFeedModuleCandidateDecorator.scala @@ -0,0 +1,104 @@ +package com.twitter.home_mixer.functional_component.decorator + +import com.twitter.home_mixer.functional_component.decorator.TuneFeedModuleCandidateDecorator.EntryNamespaceString +import com.twitter.home_mixer.functional_component.decorator.TuneFeedModuleCandidateDecorator.GrokTopicBaseUrl +import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.TuneFeedFeedbackActionInfoBuilder +import com.twitter.home_mixer.model.HomeFeatures.CurrentDisplayedGrokTopicFeature +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.icon.HorizonIconBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.StaticUrlBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.social_context.GeneralModuleSocialContextBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.ModuleStrStatic +import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.StrStatic +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleFooterBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleFooterDisplayTypeBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleHeaderBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleHeaderDisplayTypeBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.decorator.Decoration +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace +import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.FeedbackStroke +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ExternalUrl +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TextOnlyGeneralContextType +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Feedback +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.FeedbackList +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Separator +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.util.Base64UrlSafeStringEncoder +import java.nio.charset.Charset +import javax.inject.Inject +import javax.inject.Singleton + +object TuneFeedModuleCandidateDecorator { + val EntryNamespaceString = "tune-feed" + val GrokTopicBaseUrl = "" +} + +@Singleton +class TuneFeedModuleCandidateDecorator @Inject() ( + tuneFeedFeedbackActionInfoBuilder: TuneFeedFeedbackActionInfoBuilder) + extends CandidateDecorator[PipelineQuery, TweetCandidate] { + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[Decoration]] = { + + val clientEventInfoBuilder = HomeClientEventInfoBuilder[PipelineQuery, TweetCandidate]() + + val tuneFeedItemBuilder = + TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( + clientEventInfoBuilder = clientEventInfoBuilder, + feedbackActionInfoBuilder = Some(tuneFeedFeedbackActionInfoBuilder) + ) + + val tuneFeedItemDecorator = + UrtItemCandidateDecorator(tuneFeedItemBuilder) + + val tuneFeedModuleBuilder = { + val generalModuleSocialContextBuilder = GeneralModuleSocialContextBuilder( + textBuilder = ModuleStrStatic(text = "Tune your feed"), + contextType = TextOnlyGeneralContextType + ) + + // Safe .get, enforced in TuneFeedModuleGate + val topicId = query.features.get.getOrElse(CurrentDisplayedGrokTopicFeature, None).get._1 + val topicUrl = GrokTopicBaseUrl + + val tuneFeedModuleHeaderBuilder = ModuleHeaderBuilder( + textBuilder = StrStatic(text = query.features.get + .getOrElse(CurrentDisplayedGrokTopicFeature, None).map(_._2).getOrElse( + "Help us show you more of what you love")), + moduleSocialContextBuilder = Some(generalModuleSocialContextBuilder), + moduleHeaderIconBuilder = Some(HorizonIconBuilder(FeedbackStroke)), + moduleHeaderDisplayTypeBuilder = ModuleHeaderDisplayTypeBuilder(Feedback), + urlBuilder = Some(StaticUrlBuilder(topicUrl, ExternalUrl)), + isSticky = Some(false) + ) + + val tuneFeedModuleFooterBuilder = ModuleFooterBuilder( + textBuilder = StrStatic(text = ""), + urlBuilder = None, + moduleFooterDisplayTypeBuilder = ModuleFooterDisplayTypeBuilder(Separator) + ) + + TimelineModuleBuilder( + entryNamespace = EntryNamespace(EntryNamespaceString), + clientEventInfoBuilder = clientEventInfoBuilder, + displayTypeBuilder = StaticModuleDisplayTypeBuilder(FeedbackList), + headerBuilder = Some(tuneFeedModuleHeaderBuilder), + footerBuilder = Some(tuneFeedModuleFooterBuilder) + ) + } + UrtItemInModuleDecorator(tuneFeedItemDecorator, tuneFeedModuleBuilder) + .apply(query, candidates) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TweetCarouselModuleCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TweetCarouselModuleCandidateDecorator.scala new file mode 100644 index 000000000..0b2331360 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TweetCarouselModuleCandidateDecorator.scala @@ -0,0 +1,199 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselType.BookmarkedTweets +import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselType.CarouselType +import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselType.PinnedTweets +import com.twitter.home_mixer.param.HomeGlobalParams.EnableLandingPage +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleHeaderBuilder +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace +import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.Frown +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.CondensedTweet +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Carousel +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Classic +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleHeader +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.stringcenter.client.core.ExternalString +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +trait ClientComponents { + val clientEventComponent: String + val entryNamespaceString: String + val headerLink: Option[String] = None +} + +object BookmarksModuleCandidateDecorator extends ClientComponents { + override val clientEventComponent = "bookmarked-tweet" + override val entryNamespaceString = "bookmarked-tweet" + override val headerLink = Some("") +} + +object PinnedTweetsModuleCandidateDecorator extends ClientComponents { + override val clientEventComponent = "pinned-tweets" + override val entryNamespaceString = "pinned-tweets" + override val headerLink = Some("") +} + +object TweetCarouselType extends Enumeration { + type CarouselType = Value + val BookmarkedTweets, PinnedTweets = Value +} + +@Singleton +class TweetCarouselModuleCandidateDecorator @Inject() ( + homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, + @ProductScoped stringCenterProvider: Provider[StringCenter], + externalStrings: HomeMixerExternalStrings, + feedbackStrings: FeedbackStrings) { + + private val stringCenter = stringCenterProvider.get() + + private def getClientComponents(carouselType: CarouselType): ClientComponents = { + carouselType match { + case BookmarkedTweets => BookmarksModuleCandidateDecorator + case PinnedTweets => PinnedTweetsModuleCandidateDecorator + } + } + + def build( + carouselType: CarouselType + ): UrtItemInModuleDecorator[PipelineQuery, TweetCandidate, Nothing] = { + val components = getClientComponents(carouselType) + + val clientEventInfoBuilder = HomeClientEventInfoBuilder[PipelineQuery, TweetCandidate]() + + val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( + clientEventInfoBuilder = clientEventInfoBuilder, + displayType = CondensedTweet, + timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), + feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) + ) + + val tweetItemDecorator = UrtItemCandidateDecorator(tweetItemBuilder) + + val headerBuilder = + TweetCarouselModuleHeaderBuilder(carouselType, components, stringCenter, externalStrings) + + val tweetCarouselModuleBuilder = TimelineModuleBuilder( + entryNamespace = EntryNamespace(components.entryNamespaceString), + clientEventInfoBuilder = clientEventInfoBuilder, + displayTypeBuilder = StaticModuleDisplayTypeBuilder(Carousel), + headerBuilder = Some(headerBuilder), + footerBuilder = None, + feedbackActionInfoBuilder = Some( + FeedbackActionInfoBuilder( + seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, + seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenFeedbackString, + stringCenter = stringCenter, + encodedFeedbackRequest = None + ) + ), + showMoreBehaviorBuilder = None + ) + + UrtItemInModuleDecorator(tweetItemDecorator, tweetCarouselModuleBuilder) + } +} + +case class TweetCarouselModuleHeaderBuilder( + carouselType: CarouselType, + components: ClientComponents, + stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends BaseModuleHeaderBuilder[PipelineQuery, TweetCandidate] { + + private def getLandingUrl(query: PipelineQuery): Option[Url] = { + val landingPageUrl = components.headerLink.map { url => Url(ExternalUrl, url) } + + carouselType match { + case BookmarkedTweets => landingPageUrl + case PinnedTweets if query.params(EnableLandingPage) => landingPageUrl + case _ => None + } + } + + def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Option[ModuleHeader] = { + candidates.headOption.map { candidate => + val title = carouselType match { + case BookmarkedTweets => + Str(externalStrings.BookmarksHeaderString, stringCenter) + .apply(query, candidate.candidate, candidate.features) + case PinnedTweets => + Str(externalStrings.PinnedTweetsHeaderString, stringCenter) + .apply(query, candidate.candidate, candidate.features) + } + + val landingUrl = getLandingUrl(query) + + ModuleHeader( + text = title, + sticky = Some(false), + customIcon = None, + socialContext = None, + icon = None, + moduleHeaderDisplayType = Classic, + landingUrl = landingUrl + ) + } + } +} + +case class FeedbackActionInfoBuilder( + seeLessOftenFeedbackString: ExternalString, + seeLessOftenConfirmationFeedbackString: ExternalString, + stringCenter: StringCenter, + encodedFeedbackRequest: Option[String]) + extends BaseFeedbackActionInfoBuilder[PipelineQuery, TweetCandidate] { + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + candidateFeatures: FeatureMap + ): Option[FeedbackActionInfo] = Some( + FeedbackActionInfo( + feedbackActions = Seq( + FeedbackAction( + feedbackType = SeeFewer, + prompt = Some( + Str(seeLessOftenFeedbackString, stringCenter, None) + .apply(query, candidate, candidateFeatures)), + confirmation = Some( + Str(seeLessOftenConfirmationFeedbackString, stringCenter, None) + .apply(query, candidate, candidateFeatures)), + childFeedbackActions = None, + feedbackUrl = None, + confirmationDisplayType = None, + clientEventInfo = None, + richBehavior = None, + subprompt = None, + icon = Some(Frown), + hasUndoAction = Some(true), + encodedFeedbackRequest = encodedFeedbackRequest + ) + ), + feedbackMetadata = None, + displayContext = None, + clientEventInfo = None + ) + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/VideoCarouselModuleCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/VideoCarouselModuleCandidateDecorator.scala new file mode 100644 index 000000000..89257a52b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/VideoCarouselModuleCandidateDecorator.scala @@ -0,0 +1,200 @@ +package com.twitter.home_mixer.functional_component.decorator + +import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.FeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.FeedbackStrings +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.model.HomeFeatures.VideoDisplayTypeFeature +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.FeatureModuleDisplayTypeBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.decorator.Decoration +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleFooterBuilder +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleHeaderBuilder +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.Media +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.MediaShort +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Carousel +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Classic +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.CompactCarousel +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.MediaHighCarousel +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleFooter +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleHeader +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stitch.Stitch +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelines.configapi.Param +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +object VideoCarouselModuleCandidateDecorator { + val entryNamespaceString = "video-carousel" + val deepLink = "https://twitter.com/i/video" +} + +@Singleton +class VideoCarouselModuleCandidateDecorator @Inject() ( + homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, + @ProductScoped stringCenterProvider: Provider[StringCenter], + externalStrings: HomeMixerExternalStrings, + feedbackStrings: FeedbackStrings) { + import VideoCarouselModuleCandidateDecorator._ + + private val stringCenter = stringCenterProvider.get() + + def build( + enableVideoCarouselFooter: Param[Boolean] + ): UrtItemInModuleDecorator[PipelineQuery, TweetCandidate, Nothing] = { + val clientEventInfoBuilder = HomeClientEventInfoBuilder[PipelineQuery, TweetCandidate]() + val tweetItemDecorator = + VideoItemCandidateDecorator(clientEventInfoBuilder, homeFeedbackActionInfoBuilder) + val headerBuilder = + VideoCarouselModuleHeaderBuilder( + stringCenter, + externalStrings, + deepLink, + enableVideoCarouselFooter) + + val footerBuilder = + VideoCarouselModuleFooterBuilder( + stringCenter, + externalStrings, + deepLink, + enableVideoCarouselFooter) + + val videoCarouselModuleBuilder = TimelineModuleBuilder( + entryNamespace = EntryNamespace(entryNamespaceString), + clientEventInfoBuilder = clientEventInfoBuilder, + displayTypeBuilder = + FeatureModuleDisplayTypeBuilder(VideoDisplayTypeFeature, MediaHighCarousel), + headerBuilder = Some(headerBuilder), + footerBuilder = Some(footerBuilder), + feedbackActionInfoBuilder = Some( + FeedbackActionInfoBuilder( + seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, + seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenFeedbackString, + stringCenter = stringCenter, + encodedFeedbackRequest = None + ) + ), + showMoreBehaviorBuilder = None + ) + + UrtItemInModuleDecorator(tweetItemDecorator, videoCarouselModuleBuilder) + } +} + +case class VideoItemCandidateDecorator( + clientEventInfoBuilder: HomeClientEventInfoBuilder[PipelineQuery, TweetCandidate], + homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) + extends CandidateDecorator[PipelineQuery, TweetCandidate] { + + private val MediaVideoItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( + clientEventInfoBuilder = clientEventInfoBuilder, + displayType = Media, + feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) + ) + private val MediaShortVideoItemBuilder = + TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( + clientEventInfoBuilder = clientEventInfoBuilder, + displayType = MediaShort, + feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) + ) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[Decoration]] = { + val moduleDisplayType = candidates + .flatMap { candidate => + candidate.features.getOrElse(VideoDisplayTypeFeature, None) + }.headOption.getOrElse(Carousel) + // If display type of first video is Carousel, we need a horizontal video module + // If display type of first video is CompactCarousel, we need a vertical video module + val builder = moduleDisplayType match { + case CompactCarousel => MediaShortVideoItemBuilder + case _ => MediaVideoItemBuilder + } + val candidatePresentations = candidates.map { candidate => + val itemPresentation = UrtItemPresentation( + timelineItem = builder(query, candidate.candidate, candidate.features) + ) + + Decoration(candidate.candidate, itemPresentation) + } + + Stitch.value(candidatePresentations) + } +} + +case class VideoCarouselModuleHeaderBuilder( + stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings, + deepLink: String, + enableVideoCarouselFooter: Param[Boolean]) + extends BaseModuleHeaderBuilder[PipelineQuery, TweetCandidate] { + + def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Option[ModuleHeader] = { + candidates.headOption.map { candidate => + val title = Str(externalStrings.VideoCarouselHeaderString, stringCenter) + .apply(query, candidate.candidate, candidate.features) + // If footer is enabled, do not add landing URL so footer can have the deep link + val landingUrl = if (!query.params(enableVideoCarouselFooter)) { + Some(Url(ExternalUrl, deepLink)) + } else { + None + } + + ModuleHeader( + text = title, + sticky = Some(false), + customIcon = None, + socialContext = None, + icon = None, + moduleHeaderDisplayType = Classic, + landingUrl = landingUrl + ) + } + } +} + +case class VideoCarouselModuleFooterBuilder( + stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings, + deepLink: String, + enableVideoCarouselFooter: Param[Boolean]) + extends BaseModuleFooterBuilder[PipelineQuery, TweetCandidate] { + + def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Option[ModuleFooter] = { + candidates.headOption.flatMap { candidate => + // Add footer only if footer is enabled + if (query.params(enableVideoCarouselFooter)) { + val footerText = Str(externalStrings.VideoCarouselFooterString, stringCenter) + .apply(query, candidate.candidate, candidate.features) + Some( + ModuleFooter( + text = footerText, + landingUrl = Some(Url(ExternalUrl, deepLink)) + )) + } else { + None + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel index cb59d1d73..da6611f6f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel @@ -4,23 +4,16 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/com/twitter/bijection:scrooge", - "finagle/finagle-core/src/main", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "joinkey/src/main/scala/com/twitter/joinkey/context", - "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/trends_events", "src/scala/com/twitter/suggests/controller_data", - "src/thrift/com/twitter/suggests/controller_data:controller_data-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", + "trends/trending_content/src/main/scala/com/twitter/trends/trending_content/util:compacting-number-localizer", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala index eb072f135..281678ebf 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala @@ -30,6 +30,7 @@ case class HomeAdsClientEventDetailsBuilder(injectionType: Option[String]) ControllerDataV2.HomeTweets(ht.HomeTweetsControllerData.V1(homeTweetsControllerDataV1)))) val clientEventDetails = ClientEventDetails( + aiTrendDetails = None, conversationDetails = None, timelinesDetails = Some( TimelinesDetails( @@ -38,7 +39,7 @@ case class HomeAdsClientEventDetailsBuilder(injectionType: Option[String]) sourceData = None)), articleDetails = None, liveEventDetails = None, - commerceDetails = None + commerceDetails = None, ) Some(clientEventDetails) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala index 2e4c60d25..d6afecf01 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala @@ -4,10 +4,10 @@ import com.twitter.bijection.Base64String import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.bijection.{Injection => Serializer} import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.model.HomeFeatures.PositionFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.joinkey.context.RequestJoinKeyContext +import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder import com.twitter.product_mixer.core.model.common.UniversalNoun @@ -29,11 +29,16 @@ object HomeClientEventDetailsBuilder { Serializer.connect[ControllerData, Array[Byte], Base64String, String] /** + * RequestJoinId field in HomeTweetsControllerData is repurposed to pass PredictionRequestId + * ReqeustJoinId is no longer used. If wish to switch back, uncomment the below method and + * update homeTweetsControllerDataV1.requestJoinId + * * define getRequestJoinId as a method(def) rather than a val because each new request * needs to call the context to update the id. - */ - private def getRequestJoinId(): Option[Long] = - RequestJoinKeyContext.current.flatMap(_.requestJoinId) + + * private def getRequestJoinId(): Option[Long] = + * RequestJoinKeyContext.current.flatMap(_.requestJoinId) + **/ } case class HomeClientEventDetailsBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( @@ -58,17 +63,16 @@ case class HomeClientEventDetailsBuilder[-Query <: PipelineQuery, -Candidate <: HomeTweetTypePredicates.PredicateMap, candidateFeatures) - val candidateSourceId = - candidateFeatures.getOrElse(CandidateSourceIdFeature, None).map(_.value.toByte) - val homeTweetsControllerDataV1 = v1ht.HomeTweetsControllerData( tweetTypesBitmap = tweetTypesBitmaps.getOrElse(0, 0L), tweetTypesBitmapContinued1 = tweetTypesBitmaps.get(1), - candidateTweetSourceId = candidateSourceId, traceId = Some(Trace.id.traceId.toLong), injectedPosition = candidateFeatures.getOrElse(PositionFeature, None), tweetTypesListBytes = Some(tweetTypesListBytes), - requestJoinId = getRequestJoinId(), + // Repurpose requestJoinId field to avoid adding additional payload + // Use RequestJoinId to pass PredictionRequestId for model training data join + requestJoinId = candidateFeatures.getOrElse(PredictionRequestIdFeature, None), + servedId = candidateFeatures.getOrElse(ServedIdFeature, None) ) val serializedControllerData = ControllerDataSerializer( @@ -79,12 +83,13 @@ case class HomeClientEventDetailsBuilder[-Query <: PipelineQuery, -Candidate <: conversationDetails = None, timelinesDetails = Some( TimelinesDetails( - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None).map(_.name), + injectionType = Some(candidateFeatures.get(ServedTypeFeature).name), controllerData = Some(serializedControllerData), sourceData = None)), articleDetails = None, liveEventDetails = None, - commerceDetails = None + commerceDetails = None, + aiTrendDetails = None ) Some(clientEventDetails) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.scala index f79b0d931..7dd47f860 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.scala @@ -1,14 +1,13 @@ package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.home_mixer.model.HomeFeatures.EntityTokenFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.injection.scribe.InjectionScribeUtil /** * Sets the [[ClientEventInfo]] with the `component` field set to the Suggest Type assigned to each candidate @@ -23,13 +22,11 @@ case class HomeClientEventInfoBuilder[Query <: PipelineQuery, Candidate <: Unive candidateFeatures: FeatureMap, element: Option[String] ): Option[ClientEventInfo] = { - val suggestType = candidateFeatures - .getOrElse(SuggestTypeFeature, None) - .getOrElse(throw new UnsupportedOperationException(s"No SuggestType was set")) + val servedType = candidateFeatures.get(ServedTypeFeature) Some( ClientEventInfo( - component = InjectionScribeUtil.scribeComponent(suggestType), + component = Some(servedType.originalName), element = element, details = detailsBuilder.flatMap(_.apply(query, candidate, candidateFeatures)), action = None, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.scala index dc6d51327..f8609099a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.scala @@ -25,6 +25,7 @@ case class HomeConversationModuleMetadataBuilder[ socialContext = None, enableDeduplication = Some(true) )), - gridCarouselMetadata = None + gridCarouselMetadata = None, + pillGroupMetadata = None ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala index 7272c360d..c4293e64a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala @@ -2,6 +2,8 @@ package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType @@ -11,19 +13,19 @@ import com.twitter.tweetypie.{thriftscala => tpt} object HomeTweetTypePredicates { /** - * IMPORTANT: Please avoid logging tweet types that are tied to sensitive - * internal author information / labels (e.g. blink labels, abuse labels, or geo-location). + * The predicates defined in this file are used purely for metrics tracking purposes to + * measure how often we serve posts with various attributes. */ - private[this] val CandidatePredicates: Seq[(String, FeatureMap => Boolean)] = Seq( + private val CandidatePredicates: Seq[(String, FeatureMap => Boolean)] = Seq( ("with_candidate", _ => true), ("retweet", _.getOrElse(IsRetweetFeature, false)), ("reply", _.getOrElse(InReplyToTweetIdFeature, None).nonEmpty), - ("image", _.getOrElse(EarlybirdFeature, None).exists(_.hasImage)), - ("video", _.getOrElse(EarlybirdFeature, None).exists(_.hasVideo)), + ("image", _.getOrElse(HasImageFeature, false)), + ("video", _.getOrElse(HasVideoFeature, false)), ("link", _.getOrElse(EarlybirdFeature, None).exists(_.hasVisibleLink)), ("quote", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuote.contains(true))), ("like_social_context", _.getOrElse(NonSelfFavoritedByUserIdsFeature, Seq.empty).nonEmpty), - ("protected", _.getOrElse(EarlybirdFeature, None).exists(_.isProtected)), + ("protected", _.getOrElse(AuthorIsProtectedFeature, false)), ( "has_exclusive_conversation_author_id", _.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty), @@ -56,9 +58,7 @@ object HomeTweetTypePredicates { "served_size_between_50_and_100", _.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)), ("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)), - ( - "is_self_thread_tweet", - _.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))), + ("is_self_thread_tweet", _.getOrElse(IsSelfThreadFeature, false)), ("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty), ("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)), ("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)), @@ -94,9 +94,9 @@ object HomeTweetTypePredicates { ("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)), ("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)), ("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)), - ("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)), - ("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)), - ("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)), + ("is_random_tweet", _ => false), + ("has_random_tweet_in_response", _ => false), + ("is_random_tweet_above_in_utis", _ => false), ( "has_ancestor_authored_by_viewer", candidate => @@ -116,8 +116,8 @@ object HomeTweetTypePredicates { Map.empty[String, Double]).nonEmpty), ( "tweet_age_less_than_15_seconds", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 15.seconds)), + _.getOrElse(TweetAgeFeature, None) + .exists(_ <= 15.seconds.inMillis)), ( "less_than_1_hour_since_lnpt", _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)), @@ -193,13 +193,13 @@ object HomeTweetTypePredicates { "text_only", candidate => candidate.getOrElse(HasDisplayedTextFeature, false) && - !(candidate.getOrElse(EarlybirdFeature, None).exists(_.hasImage) || - candidate.getOrElse(EarlybirdFeature, None).exists(_.hasVideo) || + !(candidate.getOrElse(HasImageFeature, false) || + candidate.getOrElse(HasVideoFeature, false) || candidate.getOrElse(EarlybirdFeature, None).exists(_.hasCard))), ( "image_only", candidate => - candidate.getOrElse(EarlybirdFeature, None).exists(_.hasImage) && + candidate.getOrElse(HasImageFeature, false) && !candidate.getOrElse(HasDisplayedTextFeature, false)), ("has_1_image", _.getOrElse(NumImagesFeature, None).exists(_ == 1)), ("has_2_images", _.getOrElse(NumImagesFeature, None).exists(_ == 2)), @@ -211,9 +211,7 @@ object HomeTweetTypePredicates { "has_liked_by_social_context", candidateFeatures => candidateFeatures - .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - .exists(candidateFeatures - .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)), + .getOrElse(ValidLikedByUserIdsFeature, Seq.empty).nonEmpty), ( "has_followed_by_social_context", _.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty), @@ -232,24 +230,323 @@ object HomeTweetTypePredicates { ("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)), ( "tweet_age_lte_30_minutes", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 30.minutes)), + _.getOrElse(TweetAgeFeature, None) + .exists(_ <= 30.minutes.inMillis)), ( "tweet_age_lte_1_hour", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 1.hour)), + _.getOrElse(TweetAgeFeature, None) + .exists(_ <= 1.hour.inMillis)), ( "tweet_age_lte_6_hours", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 6.hours)), + _.getOrElse(TweetAgeFeature, None) + .exists(_ <= 6.hours.inMillis)), ( "tweet_age_lte_12_hours", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 12.hours)), + _.getOrElse(TweetAgeFeature, None) + .exists(_ <= 12.hours.inMillis)), ( "tweet_age_gte_24_hours", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow >= 24.hours)), + _.getOrElse(TweetAgeFeature, None) + .exists(_ >= 24.hours.inMillis)), + ("author_is_blue_verified", _.getOrElse(AuthorIsBlueVerifiedFeature, false)), + ("author_is_gold_verified", _.getOrElse(AuthorIsGoldVerifiedFeature, false)), + ("author_is_gray_verified", _.getOrElse(AuthorIsGrayVerifiedFeature, false)), + ("author_is_legacy_verified", _.getOrElse(AuthorIsLegacyVerifiedFeature, false)), + ("author_is_creator", _.getOrElse(AuthorIsCreatorFeature, false)), + ( + "viral_content_creator_in_network", + candidate => + candidate.getOrElse(ViralContentCreatorFeature, false) && + candidate.getOrElse(InNetworkFeature, true)), + ( + "viral_content_creator_out_of_network", + candidate => + candidate.getOrElse(ViralContentCreatorFeature, false) && + !candidate.getOrElse(InNetworkFeature, true)), + ( + "grok_content_creator_in_network", + candidate => + candidate.getOrElse(GrokContentCreatorFeature, false) && + candidate.getOrElse(InNetworkFeature, true)), + ( + "grok_content_creator_out_of_network", + candidate => + candidate.getOrElse(GrokContentCreatorFeature, false) && + !candidate.getOrElse(InNetworkFeature, true)), + ( + "gork_content_creator_in_network", + candidate => + candidate.getOrElse(GorkContentCreatorFeature, false) && + candidate.getOrElse(InNetworkFeature, true)), + ( + "gork_content_creator_out_of_network", + candidate => + candidate.getOrElse(GorkContentCreatorFeature, false) && + !candidate.getOrElse(InNetworkFeature, true)), + ("has_location", _.getOrElse(LocationIdFeature, None).isDefined), + ("article", _.getOrElse(IsArticleFeature, false)), + ( + "grok_category_news", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_sports", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ("grok_category_entertainment", _ => false), + ( + "grok_category_business_&_finance", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_technology", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_gaming", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_movies_&_tv", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_music", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_travel", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_food", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_fashion", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_health_&_fitness", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_anime", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_celebrity", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_cryptocurrency", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_science", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_memes", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ("grok_category_art", _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_religion", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_shopping", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_cars", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_aviation", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_motorcycles", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_beauty", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_nature_&_outdoors", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_pets", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_relationships", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_home_&_garden", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_career", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_dance", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_education", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_podcasts", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ( + "grok_category_streaming", + _.getOrElse(GrokTopCategoryFeature, None).contains()), + ("grok_is_gore", _.getOrElse(GrokIsGoreFeature, None).getOrElse(false)), + ("grok_is_nsfw", _.getOrElse(GrokIsNsfwFeature, None).getOrElse(false)), + ("grok_is_spam", _.getOrElse(GrokIsSpamFeature, None).getOrElse(false)), + ("grok_is_violent", _.getOrElse(GrokIsViolentFeature, None).getOrElse(false)), + ("grok_is_low_quality", _.getOrElse(GrokIsLowQualityFeature, None).getOrElse(false)), + ("grok_is_ocr", _.getOrElse(GrokIsOcrFeature, None).getOrElse(false)), + ( + "grok_politics_neutral", // Purely for metrics tracking. Does not affect the recommendations. + _.getOrElse(GrokPoliticalInclinationFeature, None).contains(hmt.PoliticalInclination.Neutral) + ), + ( + "grok_politics_left", // Purely for metrics tracking. Does not affect the recommendations. + _.getOrElse(GrokPoliticalInclinationFeature, None).contains(hmt.PoliticalInclination.Left) + ), + ( + "grok_politics_right", // Purely for metrics tracking. Does not affect the recommendations. + _.getOrElse(GrokPoliticalInclinationFeature, None).contains(hmt.PoliticalInclination.Right) + ), + ("is_slop_lte_0", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ <= 0.0)), + ("is_slop_lte_0_2", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ <= 0.2)), + ("is_slop_gt_0", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ > 0.0)), + ("is_slop_gt_0_2", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ > 0.2)), + ("is_slop_gt_0_4", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ > 0.4)), + ("is_slop_gt_0_6", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ > 0.6)), + ( + "unique_author_ratio_lte_50_pct", + features => { + val uniqueAuthorCount = features.getOrElse(UniqueAuthorCountFeature, None).getOrElse(0) + val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(1) + uniqueAuthorCount.toDouble / servedSize <= 0.5 + } + ), + ("unique_author_lte_5", _.getOrElse(UniqueAuthorCountFeature, None).exists(_ <= 5)), + ("unique_author_lte_10", _.getOrElse(UniqueAuthorCountFeature, None).exists(_ <= 10)), + ("unique_author_lte_15", _.getOrElse(UniqueAuthorCountFeature, None).exists(_ <= 15)), + ( + "single_author_gte_25_pct", + features => { + val maxCount = features.getOrElse(MaxSingleAuthorCountFeature, None).getOrElse(0) + val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(0) + servedSize > 0 && maxCount.toDouble / servedSize >= 0.25 + }), + ( + "single_author_gte_50_pct", + features => { + val maxCount = features.getOrElse(MaxSingleAuthorCountFeature, None).getOrElse(0) + val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(0) + servedSize > 0 && maxCount.toDouble / servedSize >= 0.5 + }), + ( + "unique_category_ratio_lte_50_pct", + features => { + val uniqueAuthorCount = features.getOrElse(UniqueCategoryCountFeature, None).getOrElse(0) + val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(1) + uniqueAuthorCount.toDouble / servedSize <= 0.5 + } + ), + ("unique_category_lte_5", _.getOrElse(UniqueCategoryCountFeature, None).exists(_ <= 5)), + ("unique_category_lte_10", _.getOrElse(UniqueCategoryCountFeature, None).exists(_ <= 10)), + ("unique_category_lte_15", _.getOrElse(UniqueCategoryCountFeature, None).exists(_ <= 15)), + ( + "single_category_gte_25_pct", + features => { + val maxCount = features.getOrElse(MaxSingleCategoryCountFeature, None).getOrElse(0) + val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(0) + servedSize > 0 && maxCount.toDouble / servedSize >= 0.25 + }), + ( + "single_category_gte_50_pct", + features => { + val maxCount = features.getOrElse(MaxSingleCategoryCountFeature, None).getOrElse(0) + val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(0) + servedSize > 0 && maxCount.toDouble / servedSize >= 0.5 + }), + ("is_grokslopscore_low_1", _.getOrElse(GrokSlopScoreFeature, None).contains(1L)), + ("is_grokslopscore_med_2", _.getOrElse(GrokSlopScoreFeature, None).contains(2L)), + ("is_grokslopscore_high_3", _.getOrElse(GrokSlopScoreFeature, None).contains(3L)), + ("is_boosted", _.getOrElse(IsBoostedCandidateFeature, false)), + ( + "has_source_signal_tweet_favorite", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetFavorite"))), + ( + "has_source_signal_retweet", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("Retweet"))), + ( + "has_source_signal_reply", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("Reply"))), + ( + "has_source_signal_tweet_bookmark_v1", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetBookmarkV1"))), + ( + "has_source_signal_original_tweet", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("OriginalTweet"))), + ( + "has_source_signal_account_follow", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("AccountFollow"))), + ( + "has_source_signal_tweet_share_v1", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetShareV1"))), + ( + "has_source_signal_tweet_photo_expand", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetPhotoExpand"))), + ( + "has_source_signal_search_tweet_click", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("SearchTweetClick"))), + ( + "has_source_signal_profile_tweet_click", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("ProfileTweetClick"))), + ( + "has_source_signal_tweet_video_open", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetVideoOpen"))), + ( + "has_source_signal_video_view_90d_quality_v1_all_surfaces", + _.getOrElse(SourceSignalFeature, None) + .exists(_.signalType.contains("VideoView90dQualityV1AllSurfaces"))), + ( + "has_source_signal_video_view_90d_quality_v2", + _.getOrElse(SourceSignalFeature, None) + .exists(_.signalType.contains("VideoView90dQualityV2"))), + ( + "has_source_signal_video_view_90d_quality_v2_visibility_75", + _.getOrElse(SourceSignalFeature, None) + .exists(_.signalType.contains("VideoView90dQualityV2Visibility75"))), + ( + "has_source_signal_video_view_90d_quality_v2_visibility_100", + _.getOrElse(SourceSignalFeature, None) + .exists(_.signalType.contains("VideoView90dQualityV2Visibility100"))), + ( + "has_source_signal_video_view_90d_quality_v3", + _.getOrElse(SourceSignalFeature, None) + .exists(_.signalType.contains("VideoView90dQualityV3"))), + ( + "has_source_signal_account_block", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("AccountBlock"))), + ( + "has_source_signal_account_mute", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("AccountMute"))), + ( + "has_source_signal_tweet_report", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetReport"))), + ( + "has_source_signal_tweet_dont_like", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetDontLike"))), + ( + "has_source_signal_tweet_report_v2", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetReportV2"))), + ( + "has_source_signal_tweet_dont_like_v2", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetDontLikeV2"))), + ( + "has_source_signal_notification_open_and_click_v1", + _.getOrElse(SourceSignalFeature, None) + .exists(_.signalType.contains("NotificationOpenAndClickV1"))), + ( + "has_source_signal_feedback_notrelevant", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("FeedbackNotrelevant"))), + ( + "has_source_signal_feedback_relevant", + _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("FeedbackRelevant"))), + ( + "has_source_signal_high_quality_source_tweet", + _.getOrElse(SourceSignalFeature, None) + .exists(_.signalType.contains("HighQualitySourceTweet"))), + ( + "has_source_signal_high_quality_source_user", + _.getOrElse(SourceSignalFeature, None) + .exists(_.signalType.contains("HighQualitySourceUser"))), ) val PredicateMap = CandidatePredicates.toMap diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/KeywordTrendMetaDescriptionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/KeywordTrendMetaDescriptionBuilder.scala new file mode 100644 index 000000000..8ba134fa3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/KeywordTrendMetaDescriptionBuilder.scala @@ -0,0 +1,46 @@ +package com.twitter.home_mixer.functional_component.decorator.builder + +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.BaseTrendMetaDescriptionBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str +import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendTweetCountFeature +import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.trends.trending_content.util.CompactingNumberLocalizer +import com.twitter.stringcenter.client.StringCenter +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +class KeywordTrendMetaDescriptionBuilder @Inject() ( + externalStrings: HomeMixerExternalStrings, + @ProductScoped stringCenterProvider: Provider[StringCenter]) + extends BaseTrendMetaDescriptionBuilder[PipelineQuery, UnifiedTrendCandidate] { + + private val stringCenter = stringCenterProvider.get() + + private val tweetCountStr = Str( + text = externalStrings.KeywordTrendsTweetCountDescriptionString, + stringCenter = stringCenter + ) + + private val compactingNumberLocalizer = new CompactingNumberLocalizer() + + def apply( + query: PipelineQuery, + candidate: UnifiedTrendCandidate, + candidateFeatures: FeatureMap + ): Option[String] = { + // E.g. "23.4K posts" + candidateFeatures.getOrElse(TrendTweetCountFeature, None).map { tweetCount => + val compactedTweetCount = compactingNumberLocalizer.localizeAndCompact( + query.getLanguageCode + .getOrElse("en"), + tweetCount) + tweetCountStr(query, candidate, candidateFeatures).format(compactedTweetCount) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/VerifiedPromptBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/VerifiedPromptBuilder.scala new file mode 100644 index 000000000..0db8dca02 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/VerifiedPromptBuilder.scala @@ -0,0 +1,71 @@ +package com.twitter.home_mixer.functional_component.decorator.builder + +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.message.InlinePromptCandidateUrtItemStringCenterBuilder.InlinePromptClientEventInfoElement +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.message.MessageTextActionBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str +import com.twitter.product_mixer.component_library.model.candidate.InlinePromptCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.InlinePromptMessageContent +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessagePromptItem +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stringcenter.client.ExternalStringRegistry +import com.twitter.stringcenter.client.StringCenter + +case class VerifiedPromptBuilder( + clientEventInfoBuilder: BaseClientEventInfoBuilder[PipelineQuery, InlinePromptCandidate], + stringCenter: StringCenter, + externalStringRegistry: ExternalStringRegistry) + extends CandidateUrtEntryBuilder[ + PipelineQuery, + InlinePromptCandidate, + MessagePromptItem + ] { + private val VerifiedUrl = "" + + private val headerExternalStr = externalStringRegistry.createProdString("verified_prompt_header") + private val bodyExternalStr = externalStringRegistry.createProdString("verified_prompt_body") + private val buttonExternalStr = + externalStringRegistry.createProdString("creator_subscriptions_teaser_button") + + override def apply( + query: PipelineQuery, + candidate: InlinePromptCandidate, + candidateFeatures: FeatureMap + ): MessagePromptItem = { + val headerStr = Str(headerExternalStr, stringCenter).apply(query, candidate, candidateFeatures) + val bodyStr = Str(bodyExternalStr, stringCenter).apply(query, candidate, candidateFeatures) + val buttonStrBuilder = Str(buttonExternalStr, stringCenter) + + val button = MessageTextActionBuilder( + textBuilder = buttonStrBuilder, + dismissOnClick = false, + url = Some(VerifiedUrl) + ).apply(query, candidate, candidateFeatures) + + MessagePromptItem( + id = candidate.id, + sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase + clientEventInfo = clientEventInfoBuilder( + query = query, + candidate = candidate, + candidateFeatures = candidateFeatures, + element = Some(InlinePromptClientEventInfoElement) + ), + isPinned = None, + content = InlinePromptMessageContent( + headerText = headerStr, + bodyText = Some(bodyStr), + primaryButtonAction = Some(button), + secondaryButtonAction = None, + headerRichText = None, + bodyRichText = None, + socialContext = None, + userFacepile = None + ), + impressionCallbacks = None, + feedbackActionInfo = None + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala index dee273f5f..c00245be2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala @@ -1,7 +1,6 @@ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -25,8 +24,7 @@ case class AuthorChildFeedbackActionBuilder @Inject() ( promptExternalString = externalStrings.showFewerTweetsString, confirmationExternalString = externalStrings.showFewerTweetsConfirmationString, engagementType = t.FeedbackEngagementType.Tweet, - stringCenter = stringCenter, - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) + stringCenter = stringCenter ) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel index 3a4f19592..6e191bdd1 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel @@ -9,11 +9,10 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "src/thrift/com/twitter/timelines/service:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala index a23b57dee..8ca24a60f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala @@ -9,6 +9,7 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.icon import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser import com.twitter.product_mixer.core.product.guice.scope.ProductScoped @@ -21,7 +22,9 @@ case class BlockUserChildFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { - def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { + def apply( + candidateFeatures: FeatureMap + ): Option[ChildFeedbackAction] = { val userIdOpt = if (candidateFeatures.getOrElse(IsRetweetFeature, false)) candidateFeatures.getOrElse(SourceUserIdFeature, None) @@ -35,17 +38,30 @@ case class BlockUserChildFeedbackActionBuilder @Inject() ( externalStrings.blockUserString, Map("username" -> userScreenName) ) + val confirmation = stringCenter.prepare( + externalStrings.blockUserConfirmationString, + Map("username" -> userScreenName) + ) + ChildFeedbackAction( feedbackType = RichBehavior, prompt = Some(prompt), - confirmation = None, + confirmation = Some(confirmation), + subprompt = None, feedbackUrl = None, hasUndoAction = Some(true), confirmationDisplayType = Some(BottomSheet), - clientEventInfo = None, + clientEventInfo = Some( + ClientEventInfo( + component = None, + element = Some("block"), + details = None, + action = Some("click"), + entityToken = None + ) + ), icon = Some(icon.No), - richBehavior = Some(RichFeedbackBehaviorBlockUser(userId)), - subprompt = None + richBehavior = Some(RichFeedbackBehaviorBlockUser(userId)) ) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ChildFeedbackActionBuilder.scala new file mode 100644 index 000000000..7e741b9be --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ChildFeedbackActionBuilder.scala @@ -0,0 +1,270 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackPromptNegativeParam +import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackPromptNeutralParam +import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackPromptPositiveParam +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.stringcenter.client.core.ExternalString +import com.twitter.timelines.common.{thriftscala => tlc} +import com.twitter.timelineservice.model.FeedbackInfo +import com.twitter.timelineservice.model.FeedbackMetadata +import com.twitter.timelineservice.{thriftscala => tlst} +import javax.inject.Inject +import javax.inject.Singleton + +trait ChildFeedbackActionBuilder { + val stringCenter: StringCenter + def feedbackType: tlst.FeedbackType + def internalFeedbackType: FeedbackType + def promptString: ExternalString + def confirmationString: ExternalString + def clientEventElement: String + def clientEventAction: String + def clientEventComponent: Option[String] = None + def hasUndoAction: Boolean = true + def confirmationDisplayType: Option[ConfirmationDisplayType] = None + def getPrompt(query: PipelineQuery): String = { + stringCenter.prepare(promptString) + } + + def apply( + query: PipelineQuery, + candidate: TweetCandidate, + candidateFeatures: FeatureMap, + ): Option[ChildFeedbackAction] = { + val prompt = getPrompt(query) + val confirmation = stringCenter.prepare(confirmationString) + val feedbackMetadata = FeedbackMetadata( + engagementType = None, + entityIds = Seq(tlc.FeedbackEntity.TweetId(candidate.id)), + ttl = Some(FeedbackUtil.FeedbackTtl) + ) + val feedbackUrl = FeedbackInfo.feedbackUrl( + feedbackType = feedbackType, + feedbackMetadata = feedbackMetadata, + injectionType = None + ) + + Some( + ChildFeedbackAction( + feedbackType = internalFeedbackType, + prompt = Some(prompt), + confirmation = Some(confirmation), + feedbackUrl = Some(feedbackUrl), + hasUndoAction = Some(hasUndoAction), + confirmationDisplayType = confirmationDisplayType, + clientEventInfo = Some( + ClientEventInfo( + component = clientEventComponent, + element = Some(clientEventElement), + details = None, + action = Some(clientEventAction), + entityToken = None + ) + ), + icon = None, + richBehavior = None, + subprompt = None + ) + ) + } +} + +@Singleton +case class SeeMoreChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Relevant + override def promptString: ExternalString = externalStrings.seeMoreString + override def confirmationString: ExternalString = + externalStrings.genericConfirmationString + override def clientEventComponent: Option[String] = Some("for_you_post_followup") + override def clientEventElement: String = "feedback_relevant" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = Relevant + override def hasUndoAction: Boolean = false + override def confirmationDisplayType: Option[ConfirmationDisplayType] = Some(BottomSheet) +} + +@Singleton +case class SeeLessChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.NotRelevant + override def promptString: ExternalString = externalStrings.seeLessString + override def confirmationString: ExternalString = + externalStrings.genericConfirmationString + override def clientEventComponent: Option[String] = Some("for_you_post_followup") + override def clientEventElement: String = "feedback_notrelevant" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = NotRelevant + override def hasUndoAction: Boolean = false + override def confirmationDisplayType: Option[ConfirmationDisplayType] = Some(BottomSheet) +} + +@Singleton +case class RelevantChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Relevant + override def promptString: ExternalString = externalStrings.relevantString + override def confirmationString: ExternalString = + externalStrings.relevantConfirmationString + override def clientEventComponent: Option[String] = Some("for_you_post_relevance_prompt") + override def clientEventElement: String = "feedback_relevant" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = Relevant + override def hasUndoAction: Boolean = false + override def confirmationDisplayType: Option[ConfirmationDisplayType] = Some(BottomSheet) + override def getPrompt(query: PipelineQuery): String = { + query.params(PostFeedbackPromptPositiveParam) + } +} + +@Singleton +case class NotRelevantChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.NotRelevant + override def promptString: ExternalString = externalStrings.notRelevantString + override def confirmationString: ExternalString = + externalStrings.notRelevantConfirmationString + override def clientEventComponent: Option[String] = Some("for_you_post_relevance_prompt") + override def clientEventElement: String = "feedback_notrelevant" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = NotRelevant + override def hasUndoAction: Boolean = false + override def confirmationDisplayType: Option[ConfirmationDisplayType] = Some(BottomSheet) + override def getPrompt(query: PipelineQuery): String = { + query.params(PostFeedbackPromptNegativeParam) + } + +} + +@Singleton +case class NeutralChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Neutral + override def promptString: ExternalString = externalStrings.neutralString + override def confirmationString: ExternalString = + externalStrings.neutralConfirmationString + override def clientEventComponent: Option[String] = Some("for_you_post_relevance_prompt") + override def clientEventElement: String = "feedback_neutral" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = Neutral + override def hasUndoAction: Boolean = false + override def confirmationDisplayType: Option[ConfirmationDisplayType] = Some(BottomSheet) + override def getPrompt(query: PipelineQuery): String = { + query.params(PostFeedbackPromptNeutralParam) + } +} + +@Singleton +case class DontlikeNotRelevantChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.NotRelevant + override def promptString: ExternalString = externalStrings.notRelevantString + override def confirmationString: ExternalString = + externalStrings.notRelevantConfirmationString + override def clientEventElement: String = "feedback_notrelevant" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = NotRelevant +} + +@Singleton +case class HatefulChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Hateful + override def promptString: ExternalString = externalStrings.hatefulString + override def confirmationString: ExternalString = + externalStrings.hatefulConfirmationString + override def clientEventElement: String = "feedback_hateful" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = Hateful +} + +@Singleton +case class BoringChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Boring + override def promptString: ExternalString = externalStrings.boringString + override def confirmationString: ExternalString = + externalStrings.boringConfirmationString + override def clientEventElement: String = "feedback_boring" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = Boring +} + +@Singleton +case class ConfusingChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Confusing + override def promptString: ExternalString = externalStrings.confusingString + override def confirmationString: ExternalString = + externalStrings.confusingConfirmationString + override def clientEventElement: String = "feedback_confusing" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = Confusing +} + +@Singleton +case class ClickbaitChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Clickbait + override def promptString: ExternalString = externalStrings.clickbaitString + override def confirmationString: ExternalString = + externalStrings.clickbaitConfirmationString + override def clientEventElement: String = "feedback_clickbait" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = Clickbait +} + +@Singleton +case class RagebaitChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Ragebait + override def promptString: ExternalString = externalStrings.ragebaitString + override def confirmationString: ExternalString = + externalStrings.ragebaitConfirmationString + override def clientEventElement: String = "feedback_ragebait" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = Ragebait +} + +@Singleton +case class RegretChildFeedbackActionBuilder @Inject() ( + @ProductScoped override val stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) + extends ChildFeedbackActionBuilder { + override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Regret + override def promptString: ExternalString = externalStrings.regretString + override def confirmationString: ExternalString = + externalStrings.regretConfirmationString + override def clientEventElement: String = "feedback_regret" + override def clientEventAction: String = "click" + override def internalFeedbackType: FeedbackType = Regret +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DebugSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DebugSocialContextBuilder.scala new file mode 100644 index 000000000..43e158ba3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DebugSocialContextBuilder.scala @@ -0,0 +1,55 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableDebugString +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.CommunityGeneralContextType +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DeepLink +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.util.Try + +object DebugSocialContextBuilder extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { + + val TweetUrl = "" + val UserUrl = "" + val TrendsUrl = "" + val UserSignals = Set("Follow", "Profile") + val Trends = "Trends" + + def apply( + query: PipelineQuery, + candidate: TweetCandidate, + candidateFeatures: FeatureMap + ): Option[SocialContext] = { + if (query.params(EnableDebugString)) { + candidateFeatures.getOrElse(DebugStringFeature, None).map { debugString => + val signalId = Try(debugString.split(" ").head.toLong).toOption + val baseUrl = + if (UserSignals.exists(debugString.contains)) UserUrl + else if (debugString.contains(Trends)) TrendsUrl + else TweetUrl + + val url = signalId.map { id => + Url( + urlType = DeepLink, + url = s"$baseUrl$id", + urtEndpointOptions = None + ) + } + + GeneralContext( + contextType = CommunityGeneralContextType, + text = debugString, + url = None, + contextImageUrls = None, + landingUrl = url + ) + } + } else None + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala index 9032b53d9..d7c7c5dd4 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala @@ -1,21 +1,23 @@ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableNahFeedbackInfoParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdditionalChildFeedbackParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnableBlockMuteReportChildFeedbackParam import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.Frown +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DontLike import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.common.{thriftscala => tlc} -import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackMetadata +import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.{thriftscala => tls} import javax.inject.Inject import javax.inject.Singleton @@ -26,11 +28,18 @@ case class DontLikeFeedbackActionBuilder @Inject() ( externalStrings: HomeMixerExternalStrings, authorChildFeedbackActionBuilder: AuthorChildFeedbackActionBuilder, retweeterChildFeedbackActionBuilder: RetweeterChildFeedbackActionBuilder, - notRelevantChildFeedbackActionBuilder: NotRelevantChildFeedbackActionBuilder, - unfollowUserChildFeedbackActionBuilder: UnfollowUserChildFeedbackActionBuilder, - muteUserChildFeedbackActionBuilder: MuteUserChildFeedbackActionBuilder, + notRelevantChildFeedbackActionBuilder: DontlikeNotRelevantChildFeedbackActionBuilder, + hatefulChildFeedbackActionBuilder: HatefulChildFeedbackActionBuilder, + boringChildFeedbackActionBuilder: BoringChildFeedbackActionBuilder, + confusingChildFeedbackActionBuilder: ConfusingChildFeedbackActionBuilder, + clickbaitChildFeedbackActionBuilder: ClickbaitChildFeedbackActionBuilder, + ragebaitChildFeedbackActionBuilder: RagebaitChildFeedbackActionBuilder, + regretChildFeedbackActionBuilder: RegretChildFeedbackActionBuilder, blockUserChildFeedbackActionBuilder: BlockUserChildFeedbackActionBuilder, - reportTweetChildFeedbackActionBuilder: ReportTweetChildFeedbackActionBuilder) { + muteUserChildFeedbackActionBuilder: MuteUserChildFeedbackActionBuilder) { + + private val DontLikeClientEventInfo = + ClientEventInfo(None, Some("feedback_dontlike"), None, Some("click"), None) def apply( query: PipelineQuery, @@ -50,22 +59,34 @@ case class DontLikeFeedbackActionBuilder @Inject() ( val feedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = tls.FeedbackType.DontLike, feedbackMetadata = feedbackMetadata, - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) + injectionType = None ) - val childFeedbackActions = if (query.params(EnableNahFeedbackInfoParam)) { - Seq( - unfollowUserChildFeedbackActionBuilder(candidateFeatures), - muteUserChildFeedbackActionBuilder(candidateFeatures), - blockUserChildFeedbackActionBuilder(candidateFeatures), - reportTweetChildFeedbackActionBuilder(candidate) - ).flatten - } else { - Seq( - authorChildFeedbackActionBuilder(candidateFeatures), - retweeterChildFeedbackActionBuilder(candidateFeatures), - notRelevantChildFeedbackActionBuilder(candidate, candidateFeatures) - ).flatten - } + + val additionalChildFeedbackActions: Seq[ChildFeedbackAction] = + if (query.params(EnableAdditionalChildFeedbackParam)) { + Seq( + hatefulChildFeedbackActionBuilder(query, candidate, candidateFeatures), + boringChildFeedbackActionBuilder(query, candidate, candidateFeatures), + confusingChildFeedbackActionBuilder(query, candidate, candidateFeatures), + clickbaitChildFeedbackActionBuilder(query, candidate, candidateFeatures), + ragebaitChildFeedbackActionBuilder(query, candidate, candidateFeatures), + regretChildFeedbackActionBuilder(query, candidate, candidateFeatures) + ).flatten + } else Seq.empty + + val blockMuteReportChildFeedbackActions: Seq[ChildFeedbackAction] = + if (query.params(EnableBlockMuteReportChildFeedbackParam)) { + Seq( + blockUserChildFeedbackActionBuilder(candidateFeatures), + muteUserChildFeedbackActionBuilder(candidateFeatures) + ).flatten + } else Seq.empty + + val childFeedbackActions = Seq( + authorChildFeedbackActionBuilder(candidateFeatures), + retweeterChildFeedbackActionBuilder(candidateFeatures), + notRelevantChildFeedbackActionBuilder(query, candidate, candidateFeatures), + ).flatten ++ additionalChildFeedbackActions ++ blockMuteReportChildFeedbackActions FeedbackAction( feedbackType = DontLike, @@ -76,7 +97,7 @@ case class DontLikeFeedbackActionBuilder @Inject() ( feedbackUrl = Some(feedbackUrl), hasUndoAction = Some(true), confirmationDisplayType = None, - clientEventInfo = None, + clientEventInfo = Some(DontLikeClientEventInfo), icon = Some(Frown), richBehavior = None, subprompt = None, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala index 950271e4a..2e2059d80 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala @@ -16,7 +16,7 @@ object EngagerSocialContextBuilder { private val DirectInjectionContentSourceRequestParamName = "dis" private val DirectInjectionIdRequestParamName = "diid" private val DirectInjectionContentSourceSocialProofUsers = "socialproofusers" - private val SocialProofUrl = "" + private val SocialProofUrl = "/2/timeline/social_proof.json" } case class EngagerSocialContextBuilder( @@ -36,8 +36,8 @@ case class EngagerSocialContextBuilder( val realNames = candidateFeatures.getOrElse(RealNamesFeature, Map.empty[Long, String]) val validSocialContextIdAndScreenNames = socialContextIds.flatMap { socialContextId => realNames - .get(socialContextId).map(screenName => - SocialContextIdAndScreenName(socialContextId, screenName)) + .get(socialContextId) + .map(screenName => SocialContextIdAndScreenName(socialContextId, screenName)) } validSocialContextIdAndScreenNames match { @@ -47,28 +47,32 @@ case class EngagerSocialContextBuilder( Some(mkOneUserSocialContext(socialContextString, user.socialContextId)) case Seq(firstUser, secondUser) => val socialContextString = - stringCenter - .prepare( - twoUsersString, - Map("user1" -> firstUser.screenName, "user2" -> secondUser.screenName)) + stringCenter.prepare( + twoUsersString, + Map("user1" -> firstUser.screenName, "user2" -> secondUser.screenName) + ) Some( mkManyUserSocialContext( socialContextString, query.getRequiredUserId, - validSocialContextIdAndScreenNames.map(_.socialContextId))) + validSocialContextIdAndScreenNames.map(_.socialContextId) + ) + ) case firstUser +: otherUsers => val otherUsersCount = otherUsers.size val socialContextString = - stringCenter - .prepare( - moreUsersString, - Map("user" -> firstUser.screenName, "count" -> otherUsersCount)) + stringCenter.prepare( + moreUsersString, + Map("user" -> firstUser.screenName, "count" -> otherUsersCount) + ) Some( mkManyUserSocialContext( socialContextString, query.getRequiredUserId, - validSocialContextIdAndScreenNames.map(_.socialContextId))) + validSocialContextIdAndScreenNames.map(_.socialContextId) + ) + ) case _ => None } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala index 08534f26e..613876e11 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala @@ -9,7 +9,6 @@ import com.twitter.timelines.common.{thriftscala => tlc} import com.twitter.timelines.service.{thriftscala => t} import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackMetadata -import com.twitter.timelineservice.suggests.{thriftscala => st} import com.twitter.timelineservice.{thriftscala => tlst} object FeedbackUtil { @@ -22,8 +21,7 @@ object FeedbackUtil { promptExternalString: ExternalString, confirmationExternalString: ExternalString, engagementType: t.FeedbackEngagementType, - stringCenter: StringCenter, - injectionType: Option[st.SuggestType] + stringCenter: StringCenter ): Option[ChildFeedbackAction] = { namesByUserId.get(userId).map { userScreenName => val prompt = stringCenter.prepare( @@ -41,7 +39,7 @@ object FeedbackUtil { val feedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = tlst.FeedbackType.SeeFewer, feedbackMetadata = feedbackMetadata, - injectionType = injectionType + injectionType = None ) ChildFeedbackAction( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala index c932f6362..42d9c0593 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala @@ -1,24 +1,27 @@ package com.twitter.home_mixer.functional_component.decorator.urt.builder -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.param.HomeGlobalParams.EnableNahFeedbackInfoParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnablePostFeedbackParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnablePostFollowupParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnablePostDetailsNegativeFeedbackParam +import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackThresholdParam +import com.twitter.home_mixer.param.HomeGlobalParams.PostFollowupThresholdParam import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.service.{thriftscala => t} -import com.twitter.timelines.util.FeedbackMetadataSerializer import javax.inject.Inject import javax.inject.Singleton +import scala.util.Random @Singleton class HomeFeedbackActionInfoBuilder @Inject() ( - notInterestedTopicFeedbackActionBuilder: NotInterestedTopicFeedbackActionBuilder, - dontLikeFeedbackActionBuilder: DontLikeFeedbackActionBuilder) + postFeedbackActionBuilder: PostFeedbackActionBuilder, + postFollowupFeedbackActionBuilder: PostFollowupFeedbackActionBuilder, + dontLikeFeedbackActionBuilder: DontLikeFeedbackActionBuilder, + postDetailsNegativeFeedbackActionBuilder: PostDetailsNegativeFeedbackActionBuilder) extends BaseFeedbackActionInfoBuilder[PipelineQuery, TweetCandidate] { override def apply( @@ -27,24 +30,50 @@ class HomeFeedbackActionInfoBuilder @Inject() ( candidateFeatures: FeatureMap ): Option[FeedbackActionInfo] = { val supportedProduct = query.product match { - case FollowingProduct => query.params(EnableNahFeedbackInfoParam) case ForYouProduct => true case _ => false } val isAuthoredByViewer = CandidatesUtil.isAuthoredByViewer(query, candidateFeatures) if (supportedProduct && !isAuthoredByViewer) { - val feedbackActions = Seq( - notInterestedTopicFeedbackActionBuilder(candidateFeatures), - dontLikeFeedbackActionBuilder(query, candidate, candidateFeatures) - ).flatten - val feedbackMetadata = FeedbackMetadataSerializer.serialize( - t.FeedbackMetadata(injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None))) + // avoid showing post feedback for every candidate + val shouldShowPostFeedback = + query.params(EnablePostFeedbackParam) && + Random.nextDouble() < query.params(PostFeedbackThresholdParam) + + val shouldShowPostFollowup = + query.params(EnablePostFollowupParam) && + Random.nextDouble() < query.params(PostFollowupThresholdParam) + + val shouldShowPostDetailsNegative = + query.params(EnablePostDetailsNegativeFeedbackParam) + + val feedbackActions = + Seq( + if (shouldShowPostFeedback) + postFeedbackActionBuilder(query, candidate, candidateFeatures) + else None, + if (shouldShowPostFollowup) + postFollowupFeedbackActionBuilder(query, candidate, candidateFeatures) + else None, + dontLikeFeedbackActionBuilder( + query, + candidate, + candidateFeatures + ), + if (shouldShowPostDetailsNegative) + postDetailsNegativeFeedbackActionBuilder( + query, + candidate, + candidateFeatures + ) + else None + ).flatten Some( FeedbackActionInfo( feedbackActions = feedbackActions, - feedbackMetadata = Some(feedbackMetadata), + feedbackMetadata = None, displayContext = None, clientEventInfo = None )) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetContextBuilder.scala new file mode 100644 index 000000000..5e4ea71ef --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetContextBuilder.scala @@ -0,0 +1,92 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.home_mixer.model.HomeFeatures.BasketballContextFeature +import com.twitter.home_mixer.model.HomeFeatures.GenericPostContextFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseTweetContextBuilder +import com.twitter.product_mixer.core.model.marshalling.response.urt.icon +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HomeTweetContextBuilder @Inject() () + extends BaseTweetContextBuilder[PipelineQuery, TweetCandidate] { + + private val grokCom = "grok.com" + + private def mapBasketballStatus(originalStatus: Option[String]): Option[String] = { + originalStatus.flatMap { + case "Inprogress" | "Halftime" => Some("Live") + case "Closed" | "Completed" => Some("Final") + case "Created" | "Scheduled" => Some("Upcoming") + case _ => None + } + } + + private def mapPoints(status: String, points: Option[Short]): Option[Short] = { + if (status == "Live" || status == "Final") { + points + } else { + None + } + } + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + features: FeatureMap + ): Option[TweetContext] = { + features + .getOrElse(BasketballContextFeature, None).flatMap { basketballContext => + val mappedStatus = mapBasketballStatus(basketballContext.status) + + mappedStatus.map { status => + TweetContext( + contextType = ContextType.Topic, + text = "", + landingUrl = None, + contextImageUrls = None, + context = Some(TweetContextDetails.Basketball(BasketballContext( + clock = basketballContext.clock, + homeTeamScore = mapPoints(status, basketballContext.homeTeamScore), + awayTeamScore = mapPoints(status, basketballContext.awayTeamScore), + homeTeamName = basketballContext.homeTeamName, + awayTeamName = basketballContext.awayTeamName, + status = Some(status), + url = Url( + urlType = DeepLink, + url = basketballContext.url.url, + urtEndpointOptions = None + ) + ))), + icon = None + ) + } + }.orElse { + features.getOrElse(GenericPostContextFeature, None).map { genericContext => + val grokIconOpt = if (genericContext.url.url.contains(grokCom)) Some(icon.Grok) else None + + TweetContext( + contextType = ContextType.Topic, + text = genericContext.primaryText, + landingUrl = None, + contextImageUrls = None, + context = Some(TweetContextDetails.Generic(GenericContext( + primaryText = genericContext.primaryText, + secondaryText = genericContext.secondaryText, + url = Url( + urlType = DeepLink, + url = genericContext.url.url, + urtEndpointOptions = None + ) + ))), + icon = grokIconOpt + ) + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala index 5138f254a..1505b66aa 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala @@ -2,7 +2,9 @@ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableCommunitiesContextParam import com.twitter.home_mixer.param.HomeGlobalParams.EnableSocialContextParam +import com.twitter.product_mixer.component_library.decorator.urt.builder.social_context.CommunitiesSocialContextBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder @@ -14,13 +16,10 @@ import javax.inject.Singleton @Singleton case class HomeTweetSocialContextBuilder @Inject() ( likedBySocialContextBuilder: LikedBySocialContextBuilder, - listsSocialContextBuilder: ListsSocialContextBuilder, + servedTypeSocialContextBuilder: ServedTypeSocialContextBuilder, followedBySocialContextBuilder: FollowedBySocialContextBuilder, - topicSocialContextBuilder: TopicSocialContextBuilder, extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder, - receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder, - popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder, - popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder) + receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { def apply( @@ -28,20 +27,24 @@ case class HomeTweetSocialContextBuilder @Inject() ( candidate: TweetCandidate, features: FeatureMap ): Option[SocialContext] = { + val communitiesSocialContextBuilder = + if (query.params(EnableCommunitiesContextParam)) + CommunitiesSocialContextBuilder(query, candidate, features) + else None + if (query.params(EnableSocialContextParam)) { features.getOrElse(ConversationModuleFocalTweetIdFeature, None) match { case None => - likedBySocialContextBuilder(query, candidate, features) + DebugSocialContextBuilder(query, candidate, features) + .orElse(communitiesSocialContextBuilder) + .orElse(servedTypeSocialContextBuilder(query, candidate, features)) .orElse(followedBySocialContextBuilder(query, candidate, features)) - .orElse(topicSocialContextBuilder(query, candidate, features)) - .orElse(popularVideoSocialContextBuilder(query, candidate, features)) - .orElse(listsSocialContextBuilder(query, candidate, features)) - .orElse(popularInYourAreaSocialContextBuilder(query, candidate, features)) case Some(_) => val conversationId = features.getOrElse(ConversationModuleIdFeature, None) // Only hydrate the social context into the root tweet in a conversation module if (conversationId.contains(candidate.id)) { - extendedReplySocialContextBuilder(query, candidate, features) + communitiesSocialContextBuilder + .orElse(extendedReplySocialContextBuilder(query, candidate, features)) .orElse(receivedReplySocialContextBuilder(query, candidate, features)) } else None } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala index fbc65be13..f7de858ba 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala @@ -1,7 +1,6 @@ package com.twitter.home_mixer.functional_component.decorator.urt.builder -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.ValidLikedByUserIdsFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -39,11 +38,7 @@ case class LikedBySocialContextBuilder @Inject() ( ): Option[SocialContext] = { // Liked by users are valid only if they pass both the SGS and Perspective filters. - val validLikedByUserIds = - candidateFeatures - .getOrElse(SGSValidLikedByUserIdsFeature, Nil) - .filter( - candidateFeatures.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Nil).toSet.contains) + val validLikedByUserIds = candidateFeatures.getOrElse(ValidLikedByUserIdsFeature, Nil) engagerSocialContextBuilder( socialContextIds = validLikedByUserIds, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala index 542ed8a67..f62058a58 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala @@ -7,7 +7,9 @@ import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.icon +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser import com.twitter.product_mixer.core.product.guice.scope.ProductScoped @@ -36,17 +38,29 @@ case class MuteUserChildFeedbackActionBuilder @Inject() ( externalStrings.muteUserString, Map("username" -> userScreenName) ) + val confirmation = stringCenter.prepare( + externalStrings.muteUserConfirmationString, + Map("username" -> userScreenName) + ) ChildFeedbackAction( feedbackType = RichBehavior, prompt = Some(prompt), - confirmation = None, + confirmation = Some(confirmation), + subprompt = None, feedbackUrl = None, hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, + confirmationDisplayType = Some(BottomSheet), + clientEventInfo = Some( + ClientEventInfo( + component = None, + element = Some("mute"), + details = None, + action = Some("click"), + entityToken = None + ) + ), icon = Some(icon.SpeakerOff), richBehavior = Some(RichFeedbackBehaviorToggleMuteUser(userId)), - subprompt = None ) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostDetailsNegativeFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostDetailsNegativeFeedbackActionBuilder.scala new file mode 100644 index 000000000..cb1db2897 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostDetailsNegativeFeedbackActionBuilder.scala @@ -0,0 +1,69 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.Frown +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.NotRelevant +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelines.common.{thriftscala => tlc} +import com.twitter.timelineservice.model.FeedbackInfo +import com.twitter.timelineservice.model.FeedbackMetadata +import com.twitter.timelineservice.{thriftscala => tls} + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class PostDetailsNegativeFeedbackActionBuilder @Inject() ( + @ProductScoped stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings) { + + private val PostDetailsNegativeClientEventInfo = + ClientEventInfo(None, Some("feedback_notrelevant"), None, Some("click"), None) + + def apply( + query: PipelineQuery, + candidate: TweetCandidate, + candidateFeatures: FeatureMap + ): Option[FeedbackAction] = { + CandidatesUtil.getOriginalAuthorId(candidateFeatures).map { authorId => + val feedbackEntities = Seq( + tlc.FeedbackEntity.TweetId(candidate.id), + tlc.FeedbackEntity.UserId(authorId) + ) + val feedbackMetadata = FeedbackMetadata( + engagementType = None, + entityIds = feedbackEntities, + ttl = Some(30.days) + ) + val feedbackUrl = FeedbackInfo.feedbackUrl( + feedbackType = tls.FeedbackType.NotRelevant, + feedbackMetadata = feedbackMetadata, + injectionType = None + ) + + FeedbackAction( + feedbackType = NotRelevant, + prompt = Some(stringCenter.prepare(externalStrings.notRelevantString)), + confirmation = Some(stringCenter.prepare(externalStrings.notRelevantConfirmationString)), + childFeedbackActions = None, + feedbackUrl = Some(feedbackUrl), + hasUndoAction = Some(false), + confirmationDisplayType = Some(BottomSheet), + clientEventInfo = Some(PostDetailsNegativeClientEventInfo), + icon = Some(Frown), + richBehavior = None, + subprompt = None, + encodedFeedbackRequest = None + ) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostFeedbackActionBuilder.scala new file mode 100644 index 000000000..d761dd545 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostFeedbackActionBuilder.scala @@ -0,0 +1,90 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackPromptTitleParam +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.Smile +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Generic +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelines.common.{thriftscala => tlc} +import com.twitter.timelineservice.model.FeedbackInfo +import com.twitter.timelineservice.model.FeedbackMetadata +import com.twitter.timelineservice.{thriftscala => tls} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class PostFeedbackActionBuilder @Inject() ( + @ProductScoped stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings, + relevantChildFeedbackActionBuilder: RelevantChildFeedbackActionBuilder, + notRelevantChildFeedbackActionBuilder: NotRelevantChildFeedbackActionBuilder, + neutralChildFeedbackActionBuilder: NeutralChildFeedbackActionBuilder) { + + val ClientEventInfoComponent: String = "for_you_post_relevance_prompt" + val ClientEventInfoElement: String = "relevance_prompt" + + def apply( + query: PipelineQuery, + candidate: TweetCandidate, + candidateFeatures: FeatureMap + ): Option[FeedbackAction] = { + CandidatesUtil.getOriginalAuthorId(candidateFeatures).map { authorId => + val feedbackEntities = Seq( + tlc.FeedbackEntity.TweetId(candidate.id), + tlc.FeedbackEntity.UserId(authorId) + ) + val feedbackMetadata = FeedbackMetadata( + engagementType = None, + entityIds = feedbackEntities, + ttl = Some(30.days) + ) + val feedbackUrl = FeedbackInfo.feedbackUrl( + feedbackType = tls.FeedbackType.Generic, + feedbackMetadata = feedbackMetadata, + injectionType = None + ) + + val childFeedbackActions: Seq[ChildFeedbackAction] = { + Seq( + relevantChildFeedbackActionBuilder(query, candidate, candidateFeatures), + notRelevantChildFeedbackActionBuilder(query, candidate, candidateFeatures), + // neutralChildFeedbackActionBuilder(query, candidate, candidateFeatures) + ).flatten + } + + FeedbackAction( + feedbackType = Generic, + prompt = Some(query.params(PostFeedbackPromptTitleParam)), + confirmation = Some( + stringCenter.prepare(externalStrings.genericConfirmationString) + ), + childFeedbackActions = Some(childFeedbackActions), + feedbackUrl = Some(feedbackUrl), + hasUndoAction = None, + confirmationDisplayType = Some(BottomSheet), + clientEventInfo = Some( + ClientEventInfo( + component = Some(ClientEventInfoComponent), + element = Some(ClientEventInfoElement), + details = None, + action = None, + entityToken = None + )), + icon = Some(Smile), + richBehavior = None, + subprompt = None, + encodedFeedbackRequest = None + ) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostFollowupFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostFollowupFeedbackActionBuilder.scala new file mode 100644 index 000000000..f5c41e73e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostFollowupFeedbackActionBuilder.scala @@ -0,0 +1,87 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackPromptTitleParam +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GiveFeedback +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelines.common.{thriftscala => tlc} +import com.twitter.timelineservice.model.FeedbackInfo +import com.twitter.timelineservice.model.FeedbackMetadata +import com.twitter.timelineservice.{thriftscala => tls} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class PostFollowupFeedbackActionBuilder @Inject() ( + @ProductScoped stringCenter: StringCenter, + externalStrings: HomeMixerExternalStrings, + seeMoreChildFeedbackActionBuilder: SeeMoreChildFeedbackActionBuilder, + seeLessChildFeedbackActionBuilder: SeeLessChildFeedbackActionBuilder) { + + val ClientEventInfoComponent: String = "for_you_post_followup" + val ClientEventInfoElement: String = "followup" + + def apply( + query: PipelineQuery, + candidate: TweetCandidate, + candidateFeatures: FeatureMap + ): Option[FeedbackAction] = { + CandidatesUtil.getOriginalAuthorId(candidateFeatures).map { authorId => + val feedbackEntities = Seq( + tlc.FeedbackEntity.TweetId(candidate.id), + tlc.FeedbackEntity.UserId(authorId) + ) + val feedbackMetadata = FeedbackMetadata( + engagementType = None, + entityIds = feedbackEntities, + ttl = Some(30.days) + ) + val feedbackUrl = FeedbackInfo.feedbackUrl( + feedbackType = tls.FeedbackType.Generic, + feedbackMetadata = feedbackMetadata, + injectionType = None + ) + + val childFeedbackActions: Seq[ChildFeedbackAction] = { + Seq( + seeLessChildFeedbackActionBuilder(query, candidate, candidateFeatures), + seeMoreChildFeedbackActionBuilder(query, candidate, candidateFeatures) + ).flatten + } + + FeedbackAction( + feedbackType = GiveFeedback, + prompt = Some(query.params(PostFeedbackPromptTitleParam)), + confirmation = Some( + stringCenter.prepare(externalStrings.genericConfirmationString) + ), + childFeedbackActions = Some(childFeedbackActions), + feedbackUrl = Some(feedbackUrl), + hasUndoAction = None, + confirmationDisplayType = Some(BottomSheet), + clientEventInfo = Some( + ClientEventInfo( + component = Some(ClientEventInfoComponent), + element = Some(ClientEventInfoElement), + details = None, + action = None, + entityToken = None + )), + icon = None, + richBehavior = None, + subprompt = None, + encodedFeedbackRequest = None + ) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RelevancePromptCandidateUrtItemBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RelevancePromptCandidateUrtItemBuilder.scala new file mode 100644 index 000000000..158915a46 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RelevancePromptCandidateUrtItemBuilder.scala @@ -0,0 +1,98 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.relevance_prompt.RelevancePromptCandidateUrtItemStringCenterBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.StrStatic +import com.twitter.product_mixer.component_library.model.candidate.RelevancePromptCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.Compact +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.PromptItem +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Callback +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.common.{thriftscala => thriftCommon} +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.service.thriftscala.FeedbackEngagementType +import com.twitter.timelineservice.model.FeedbackInfo +import com.twitter.timelineservice.model.FeedbackMetadata +import com.twitter.timelineservice.suggests.thriftscala.SuggestType +import com.twitter.timelineservice.{thriftscala => thrift} + +object RelevancePromptCandidateUrtItemBuilder { + + val ClientEventInfoComponent: String = "for_you_relevance_prompt" + + val ClientEventBuilder = ClientEventInfoBuilder[PipelineQuery, RelevancePromptCandidate]( + component = ClientEventInfoComponent + ) + + val ConfirmationStr = "Thank you for your feedback!" +} + +case class RelevancePromptCandidateUrtItemBuilder( + titleParam: FSParam[String], + relevantPromptParam: FSParam[String], + notRelevantPromptParam: FSParam[String], + neutralPromptParam: FSParam[String], +) extends CandidateUrtEntryBuilder[ + PipelineQuery, + RelevancePromptCandidate, + PromptItem + ] { + + import RelevancePromptCandidateUrtItemBuilder._ + + override def apply( + query: PipelineQuery, + candidate: RelevancePromptCandidate, + candidateFeatures: FeatureMap + ): PromptItem = { + + val servedId: Long = query.features.flatMap(_.getOrElse(ServedIdFeature, None)).getOrElse(0L) + + // Use user 0 as a dummy entity ID until we have a proper session ID to track + val feedbackMetadata = FeedbackMetadata( + Some(FeedbackEngagementType.RelevancePrompt), + entityIds = Seq(thriftCommon.FeedbackEntity.FeedbackId(servedId)), + ttl = Some(30.days) + ) + + val positiveFeedbackUrl = FeedbackInfo.feedbackUrl( + feedbackType = thrift.FeedbackType.Relevant, + feedbackMetadata = feedbackMetadata, + injectionType = Some(SuggestType.RelevancePrompt) + ) + val negativeFeedbackUrl = FeedbackInfo.feedbackUrl( + feedbackType = thrift.FeedbackType.NotRelevant, + feedbackMetadata = feedbackMetadata, + injectionType = Some(SuggestType.RelevancePrompt) + ) + val neutralFeedbackUrl = FeedbackInfo.feedbackUrl( + feedbackType = thrift.FeedbackType.Neutral, + feedbackMetadata = feedbackMetadata, + injectionType = Some(SuggestType.RelevancePrompt) + ) + + val relevancePromptCandidateUrtItemStringCenterBuilder = + RelevancePromptCandidateUrtItemStringCenterBuilder( + clientEventInfoBuilder = ClientEventBuilder, + titleTextBuilder = StrStatic(query.params(titleParam)), + confirmationTextBuilder = StrStatic(ConfirmationStr), + isRelevantTextBuilder = StrStatic(query.params(relevantPromptParam)), + notRelevantTextBuilder = StrStatic(query.params(notRelevantPromptParam)), + displayType = Compact, + isRelevantCallback = Callback(positiveFeedbackUrl), + notRelevantCallback = Callback(negativeFeedbackUrl), + neutralTextBuilder = Some(StrStatic(query.params(neutralPromptParam))), + neutralCallback = Some(Callback(neutralFeedbackUrl)) + ) + + relevancePromptCandidateUrtItemStringCenterBuilder( + query = query, + relevancePromptCandidate = candidate, + candidateFeatures = candidateFeatures + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala index a4ef0cbb2..4ef1a384d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala @@ -4,6 +4,7 @@ import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.marshalling.response.urt.icon import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet import com.twitter.product_mixer.core.product.guice.scope.ProductScoped @@ -17,7 +18,7 @@ case class ReportTweetChildFeedbackActionBuilder @Inject() ( externalStrings: HomeMixerExternalStrings) { def apply( - candidate: TweetCandidate + candidate: TweetCandidate, ): Option[ChildFeedbackAction] = { Some( ChildFeedbackAction( @@ -27,7 +28,15 @@ case class ReportTweetChildFeedbackActionBuilder @Inject() ( feedbackUrl = None, hasUndoAction = Some(true), confirmationDisplayType = None, - clientEventInfo = None, + clientEventInfo = Some( + ClientEventInfo( + component = None, + element = Some("report_tweet"), + details = None, + action = Some("click"), + entityToken = None + ) + ), icon = Some(icon.Flag), richBehavior = Some(RichFeedbackBehaviorReportTweet(candidate.id)), subprompt = None diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala index 006e93b58..2a3b3f4d6 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala @@ -3,7 +3,6 @@ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction @@ -29,8 +28,7 @@ case class RetweeterChildFeedbackActionBuilder @Inject() ( promptExternalString = externalStrings.showFewerRetweetsString, confirmationExternalString = externalStrings.showFewerRetweetsConfirmationString, engagementType = t.FeedbackEngagementType.Retweet, - stringCenter = stringCenter, - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) + stringCenter = stringCenter ) } } else None diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ServedTypeSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ServedTypeSocialContextBuilder.scala new file mode 100644 index 000000000..085f769d2 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ServedTypeSocialContextBuilder.scala @@ -0,0 +1,67 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.{thriftscala => t} +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityNameFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * Builds social context for tweets that have a 1:1 relationship + * between served type and social context. e.g. Lists/Communities + */ +@Singleton +case class ServedTypeSocialContextBuilder @Inject() ( + externalStrings: HomeMixerExternalStrings, + @ProductScoped stringCenterProvider: Provider[StringCenter]) + extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { + + private val ListsUrl = "" + private val CommunitiesUrl = "" + + private val stringCenter = stringCenterProvider.get() + private val listString = externalStrings.ownedSubscribedListsModuleHeaderString + private val popularGeoString = externalStrings.socialContextPopularInYourAreaString + + def apply( + query: PipelineQuery, + candidate: TweetCandidate, + candidateFeatures: FeatureMap + ): Option[SocialContext] = candidateFeatures.get(ServedTypeFeature) match { + + case t.ServedType.ForYouCommunity => + val communityId = candidateFeatures.getOrElse(CommunityIdFeature, None) + val communityName = candidateFeatures.getOrElse(CommunityNameFeature, None) + val context = GeneralContext( + contextType = CommunityGeneralContextType, + text = communityName.getOrElse(""), + url = None, + contextImageUrls = None, + landingUrl = communityId.map(id => Url(ExternalUrl, CommunitiesUrl + id)) + ) + Some(context) + + case t.ServedType.ForYouPopularGeo => + val context = GeneralContext( + contextType = LocationGeneralContextType, + text = stringCenter.prepare(popularGeoString), + url = None, + contextImageUrls = None, + landingUrl = None + ) + Some(context) + + case _ => None + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TuneFeedFeedbackActionInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TuneFeedFeedbackActionInfoBuilder.scala new file mode 100644 index 000000000..00c5cf736 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TuneFeedFeedbackActionInfoBuilder.scala @@ -0,0 +1,32 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TuneFeedFeedbackActionInfoBuilder @Inject() ( + postFollowupFeedbackActionBuilder: PostFollowupFeedbackActionBuilder, + dontLikeFeedbackActionBuilder: DontLikeFeedbackActionBuilder) + extends BaseFeedbackActionInfoBuilder[PipelineQuery, TweetCandidate] { + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + candidateFeatures: FeatureMap + ): Option[FeedbackActionInfo] = { + Some( + FeedbackActionInfo( + feedbackActions = Seq( + postFollowupFeedbackActionBuilder(query, candidate, candidateFeatures), + dontLikeFeedbackActionBuilder(query, candidate, candidateFeatures)).flatten, + feedbackMetadata = None, + displayContext = None, + clientEventInfo = None + )) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AncestorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AncestorFeatureHydrator.scala new file mode 100644 index 000000000..f10bb4295 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AncestorFeatureHydrator.scala @@ -0,0 +1,65 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} +import com.twitter.tweetconvosvc.{thriftscala => tcs} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class AncestorFeatureHydrator @Inject() ( + conversationServiceClient: tcs.ConversationService.MethodPerEndpoint) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Ancestor") + + override val features: Set[Feature[_, _]] = Set(AncestorsFeature) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val candidatesWithReplies = candidates.collect { + case candidate if candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined => + candidate.candidate.id + } + val candidateIsReplyIndexMap = candidatesWithReplies.zipWithIndex.toMap + val ancestorsRequest = tcs.GetAncestorsRequest(candidatesWithReplies) + conversationServiceClient.getAncestors(ancestorsRequest).map { getAncestorsResponse => + candidates.map { candidate => + val resultIndex = candidateIsReplyIndexMap.get(candidate.candidate.id) + val ancestors = resultIndex + .map { index => + getAncestorsResponse.ancestors(index) match { + case tcs.TweetAncestorsResult.TweetAncestors(ancestorsResult) + if ancestorsResult.nonEmpty => + ancestorsResult.head.ancestors ++ getTruncatedRootTweet(ancestorsResult.head) + case _ => Seq.empty + } + }.getOrElse(Seq.empty) + FeatureMap(AncestorsFeature, ancestors) + } + } + } + + private def getTruncatedRootTweet( + ancestors: ta.TweetAncestors, + ): Option[ta.TweetAncestor] = { + ancestors.conversationRootAuthorId.collect { + case rootAuthorId + if ancestors.state == ta.ReplyState.Partial && + ancestors.ancestors.last.tweetId != ancestors.conversationId => + ta.TweetAncestor(ancestors.conversationId, rootAuthorId) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorFeatureHydrator.scala new file mode 100644 index 000000000..121867858 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorFeatureHydrator.scala @@ -0,0 +1,95 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features.AuthorFeaturesAdapter +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.AuthorFeatureRepository +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer.util.ObservedKeyValueResultHandler +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.repository.KeyValueRepository +import com.twitter.servo.repository.KeyValueResult +import com.twitter.stitch.Stitch +import com.twitter.timelines.author_features.v1.{thriftjava => af} +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object AuthorFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class AuthorFeatureHydrator @Inject() ( + @Named(AuthorFeatureRepository) client: KeyValueRepository[Seq[Long], Long, af.AuthorFeatures], + override val statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with ObservedKeyValueResultHandler { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("AuthorFeature") + + override val features: Set[Feature[_, _]] = Set(AuthorFeature) + + override val statScope: String = identifier.toString + + private val DefaultDataRecord = new DataRecord() + + private val DefaultFeatureMap = FeatureMap(AuthorFeature, DefaultDataRecord) + + private val authorIdsCountStat = statsReceiver.scope(statScope).stat("authorIdsSize") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val inNetworkCandidates = + candidates.filter(_.features.getOrElse(FromInNetworkSourceFeature, false)) + val possiblyAuthorIds = inNetworkCandidates.map(extractKey) + val authorIds = possiblyAuthorIds.flatten.distinct + authorIdsCountStat.add(authorIds.size) + + val response: Future[KeyValueResult[Long, DataRecord]] = + if (authorIds.nonEmpty) + client(authorIds) + .map { + _.mapFound { + AuthorFeaturesAdapter.adaptToDataRecords(_).asScala.head + } + } + else Future.value(KeyValueResult.empty) + + response.map { result => + candidates.map { candidate => + val authorId = extractKey(candidate) + val authorDR = observedGet(key = authorId, keyValueResult = result) + authorDR.toOption + .flatMap { + _.map { features => + FeatureMap(AuthorFeature, features) + } + }.getOrElse(DefaultFeatureMap) + } + } + } + + private def extractKey( + candidate: CandidateWithFeatures[TweetCandidate] + ): Option[Long] = { + CandidatesUtil.getOriginalAuthorId(candidate.features) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorLargeEmbeddingsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorLargeEmbeddingsFeatureHydrator.scala new file mode 100644 index 000000000..723ca58bb --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorLargeEmbeddingsFeatureHydrator.scala @@ -0,0 +1,148 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.AuthorLargeEmbeddingsFeature +import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.AuthorLargeEmbeddingsKeyFeature +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam +import com.twitter.home_mixer_features.{thriftscala => hmf} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.adapters.large_embeddings.AuthorLargeEmbeddingsAdapter +import com.twitter.timelines.prediction.adapters.large_embeddings.HashingFeatureParams +import com.twitter.timelines.prediction.adapters.large_embeddings.HomeMixerLargeEmbeddingsFeatureHydrator +import com.twitter.timelines.prediction.adapters.large_embeddings.LargeEmbeddingsAdapter +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class AuthorLargeEmbeddingsFeatureHydrator @Inject() ( + statsReceiver: StatsReceiver, + override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] + with HomeMixerLargeEmbeddingsFeatureHydrator { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("AuthorLargeEmbeddings") + + override val features: Set[Feature[_, _]] = + Set(AuthorLargeEmbeddingsFeature, AuthorLargeEmbeddingsKeyFeature) + + override val adapter: LargeEmbeddingsAdapter = AuthorLargeEmbeddingsAdapter + + override val cacheType: hmf.Cache = hmf.Cache.AuthorLargeEmbeddings + + override val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableLargeEmbeddingsFeatureHydrationParam) + + // Hashing Features + override val defaultHashingFeatureParams: HashingFeatureParams = HashingFeatureParams( + scales = Seq(3384241453L, 3372414709L), + biases = Seq(1649585795L, 3131243219L), + modulus = 3957384397L, + bucketSize = 3000000L, + ) + + override val modelName2HashingFeatureParams: Map[String, HashingFeatureParams] = Map( + "hr_video_prod__v3_realtime" -> HashingFeatureParams( + scales = Seq(113294449L, 601841083L), + biases = Seq(2231299001L, 841367196L), + modulus = 2343760591L, + bucketSize = 3000000L, + ), + "hr_video_prod__v2_lembeds" -> HashingFeatureParams( + scales = Seq(787140070L, 633713480L), + biases = Seq(427768658L, 911091889L), + modulus = 2888480981L, + bucketSize = 300000L, + ), + "hr_prod__v4_embeds_230M" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + "hr_prod__v5_embeds_230M_and_transformer" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + "hr_prod__v5_watchtime" -> HashingFeatureParams( + scales = Seq(2328530078L, 2844016377L), + biases = Seq(1352496802L, 3011003330L), + modulus = 3979826519L, + bucketSize = 30000000L, + ), + "hr_prod__v6_transformer_v2" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + "hr_prod__v6_mixed_training" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + "hr_prod__v6_transformer_v2_kafka_merge_join" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + "hr_prod__v6_transformer_v2_realtime_debias_21apr" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + ) + private val batchSize = 25 + + private def getBatchedFeatureMap( + modelName: String, + candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], + ): Future[Seq[FeatureMap]] = { + val authorIds = + candidatesBatch.map { candidate => + candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) + } + + getLargeEmbeddings(authorIds, modelName).map { responses => + responses.map { largeEmbeddingResponse => + FeatureMapBuilder() + .add(AuthorLargeEmbeddingsFeature, largeEmbeddingResponse.dataRecord) + .add(AuthorLargeEmbeddingsKeyFeature, largeEmbeddingResponse.hashedKeys) + .build() + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val modelName = query.params(ModelNameParam) + OffloadFuturePools.offloadBatchSeqToFutureSeq( + candidates, + getBatchedFeatureMap(modelName, _), + batchSize + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel index 60c104143..1319fd4ca 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel @@ -4,58 +4,117 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "configapi/configapi-decider", - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", + "3rdparty/jvm/com/knuddels/jtokkit", + "3rdparty/jvm/io/grpc:grpc-protobuf", + "3rdparty/jvm/org/scalanlp:breeze", + "hmli/hss/src/main/thrift/com/twitter/hss:thrift-scala", + "home-mixer-features/thrift/src/main/thrift:thrift-java", + "home-mixer-features/thrift/src/main/thrift:thrift-scala", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", "joinkey/src/main/scala/com/twitter/joinkey/context", "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store", + "language/thrift:types-lib-scala", + "limiter/thrift-only/src/main/thrift:thrift-scala", + "media-understanding/embeddings/src/main/thrift/com/twitter/media-understanding/embeddings:thrift-scala", + "media-understanding/video-summary/thrift/src/main/thrift:thrift-scala", + "periscope/api-proxy-thrift/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "snowflake/src/main/scala/com/twitter/snowflake/id", + "representation-scorer/server/src/main/scala/com/twitter/representationscorer/common", + "search/search-router/thrift/src/main/thrift:thrift-scala", + "servo/repo/src/main/scala", + "src/java/com/twitter/ml/api:api-base", + "src/java/com/twitter/ml/api/constant", "src/java/com/twitter/search/common/util/lang", + "src/scala/com/twitter/ml/api/util", + "src/scala/com/twitter/simclusters_v2/common", + "src/scala/com/twitter/suggests/controller_data", + "src/scala/com/twitter/timelines/prediction/adapters/large_embeddings", + "src/scala/com/twitter/timelines/prediction/adapters/real_graph", + "src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph", "src/scala/com/twitter/timelines/prediction/adapters/request_context", - "src/thrift/com/twitter/gizmoduck:thrift-scala", + "src/scala/com/twitter/timelines/prediction/adapters/twistly", + "src/scala/com/twitter/timelines/prediction/adapters/two_hop_features", + "src/scala/com/twitter/timelines/prediction/common/util", + "src/scala/com/twitter/timelines/prediction/features/common", + "src/scala/com/twitter/timelines/prediction/features/location", + "src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph", + "src/scala/com/twitter/timelines/prediction/features/simcluster", + "src/scala/com/twitter/timelines/prediction/features/time_features", + "src/scala/com/twitter/topic_recos/common", "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-result-scala", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "src/thrift/com/twitter/timelines/impression:thrift-scala", + "src/thrift/com/twitter/timelines/content_understanding/user_interests:user_interests-scala", "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", - "src/thrift/com/twitter/timelines/real_graph:real_graph-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", "src/thrift/com/twitter/user_session_store:thrift-java", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "stitch/stitch-core", - "stitch/stitch-gizmoduck", - "stitch/stitch-socialgraph", - "stitch/stitch-timelineservice", - "stitch/stitch-tweetypie", + "strato/config/columns/analytics/video:video-strato-client", + "strato/config/columns/audiencerewards/audienceRewardsService:getSuperFollowEligibility-strato-client", + "strato/config/columns/content_understanding:content_understanding-strato-client", + "strato/config/columns/content_understanding/internal/manhattan:manhattan-strato-client", + "strato/config/columns/events/experiences/basketball:basketball-strato-client", + "strato/config/columns/events/urt:urt-strato-client", + "strato/config/columns/geo/service:service-strato-client", + "strato/config/columns/heartbeat_optimizer:heartbeat_optimizer-strato-client", + "strato/config/columns/hss/user_scores/api:api-strato-client", + "strato/config/columns/language/tweet:tweet-strato-client", + "strato/config/columns/language/user:user-strato-client", + "strato/config/columns/ml/featureStore:featureStore-strato-client", + "strato/config/columns/periscope:periscope-strato-client", + "strato/config/columns/recommendations/interaction_graph/on_prem:on-prem-interaction-graph-user-features-strato-client", + "strato/config/columns/recommendations/simclusters_v2:simclusters_v2-strato-client", + "strato/config/columns/recommendations/user-signal-service:user-signal-service-strato-client", + "strato/config/columns/searchai/grok:grok-strato-client", + "strato/config/columns/searchai/storage:storage-strato-client", + "strato/config/columns/subscription-services/subscription-verification:subscription-verification-strato-client", + "strato/config/columns/trends/trip:trip-strato-client", + "strato/config/columns/tweetypie/federated:federated-strato-client", + "strato/config/columns/tweetypie/managed:managed-strato-client", + "strato/config/columns/unified-counter/service:service-strato-client", + "strato/config/columns/user-history-transformer/user-actions:user-actions-strato-client", + "strato/config/columns/videoRecommendations/twitterClip:twitterClip-strato-client", + "strato/config/columns/viewcounts:viewcounts-strato-client", + "strato/config/src/thrift/com/twitter/strato/columns/heartbeat_optimizer:heartbeat_optimizer-scala", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/store", + "timelines/src/main/scala/com/twitter/timelines/clients/strato/topics", + "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "user_session_store/src/main/scala/com/twitter/user_session_store", - "util/util-core", + "topic-social-proof/server/src/main/scala/com/twitter/tsp/stores", + "topic-social-proof/server/src/main/thrift:thrift-scala", + "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", + "tweetconvosvc/thrift/src/main/thrift:thrift-scala", + "tweetsource/common/src/main/thrift:thrift-scala", + "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + "user_history_transformer/common/src/main/scala/com/twitter/user_history_transformer/common", + "user_history_transformer/common/src/main/scala/com/twitter/user_history_transformer/util", + "user_history_transformer/domain/src/main/scala:aggregation", + "user_history_transformer/domain/src/main/scala:backbone", + "user_history_transformer/domain/src/main/scala:user-history", + "user_history_transformer/service/src/main/java/com/x/user_action_sequence", + "user_history_transformer/thrift/src/main/thrift/com/x/user_action_sequence:user_action_sequence-scala", ], exports = [ "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BasketballContextFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BasketballContextFeatureHydrator.scala new file mode 100644 index 000000000..7c1abb6f3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BasketballContextFeatureHydrator.scala @@ -0,0 +1,65 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.BasketballContextFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.param.HomeGlobalParams.BasketballTeamAccountIdsParam +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.timelines.render.{thriftscala => urt} +import com.twitter.strato.catalog.Fetch +import com.twitter.strato.generated.client.events.experiences.basketball.PostBasketballContextClientColumn + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class BasketballContextFeatureHydrator @Inject() ( + postBasketballContextClientColumn: PostBasketballContextClientColumn +) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("BasketballContext") + + override val features: Set[Feature[_, _]] = Set(BasketballContextFeature) + + private val fetcher: Fetcher[Long, Unit, urt.BasketballContext] = postBasketballContextClientColumn.fetcher + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val basketballAuthorIds = query.params(BasketballTeamAccountIdsParam) + + Stitch.collect { + candidates.map { candidate => + val servedType = candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) + val isPromoted = (servedType == hmt.ServedType.ForYouPromoted || servedType == hmt.ServedType.FollowingPromoted) + + val authorId = candidate.features.getOrElse(AuthorIdFeature, None) + val isBasketballAuthor = authorId.exists(id => basketballAuthorIds.contains(id)) + + // Skip hydration if the post is an ad or not from a basketball account + if (isPromoted || !isBasketballAuthor) { + Stitch.value(FeatureMapBuilder().add(BasketballContextFeature, None).build()) + } else { + fetcher.fetch(candidate.candidate.id, Unit).map { + case Fetch.Result(Some(basketballContext), _) => + FeatureMapBuilder().add(BasketballContextFeature, Some(basketballContext)).build() + case _ => + FeatureMapBuilder().add(BasketballContextFeature, None).build() + } + } + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BroadcastStateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BroadcastStateFeatureHydrator.scala new file mode 100644 index 000000000..6e1ecdfaf --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BroadcastStateFeatureHydrator.scala @@ -0,0 +1,70 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.catalog.Fetch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.periscope.CoreOnBroadcastClientColumn +import com.twitter.ubs.{thriftscala => ubs} +import javax.inject.Inject +import javax.inject.Singleton + +object BroadcastStateFeature extends Feature[TweetCandidate, Option[ubs.BroadcastState]] + +@Singleton +class BroadcastStateFeatureHydrator @Inject() ( + coreOnBroadcastClientColumn: CoreOnBroadcastClientColumn) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with WithDefaultFeatureMap { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("BroadcastState") + + override val features: Set[Feature[_, _]] = Set(BroadcastStateFeature) + + private val pattern = "".r + + private val fetcher: Fetcher[String, Unit, ubs.Broadcast] = coreOnBroadcastClientColumn.fetcher + + override val defaultFeatureMap: FeatureMap = FeatureMap(BroadcastStateFeature, None) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val broadcastIdMap = candidates.flatMap { candidate => + candidate.features + .getOrElse(TweetUrlsFeature, Seq.empty) + .collectFirst { case pattern(broadcastId) => broadcastId } + .map(broadcastId => candidate.candidate.id -> broadcastId) + }.toMap + + val responses = Stitch.collect { + broadcastIdMap.values.toSeq.distinct.map { broadcastId => + fetcher.fetch(broadcastId).map { + case Fetch.Result(Some(broadcast), _) if broadcast.broadcastId.nonEmpty => + Some(broadcastId -> broadcast) + case _ => None + } + } + } + + responses.map { results => + val broadcastMap = results.flatten.toMap + candidates.map { candidate => + val broadcastState = broadcastIdMap.get(candidate.candidate.id).flatMap { broadcastId => + broadcastMap.get(broadcastId).flatMap(_.state) + } + FeatureMapBuilder().add(BroadcastStateFeature, broadcastState).build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/CategoryDiversityRescoringFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/CategoryDiversityRescoringFeatureHydrator.scala new file mode 100644 index 000000000..fc2cb806c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/CategoryDiversityRescoringFeatureHydrator.scala @@ -0,0 +1,112 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.simclusters_features.SimclustersFeaturesAdapter.SimclustersSparseTweetEmbeddingsFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.CategoryDiversityRescoringWeightParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.CategoryDiversityKParam +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import scala.jdk.CollectionConverters._ + +object CategoryDiversityRescoringFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("CategoryDiversityRescoring") + + override val features: Set[Feature[_, _]] = Set(ScoreFeature) + + final val EmptyDataRecord = new DataRecord() + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { + + diversityPenalty(query, candidates).map(score => FeatureMap(ScoreFeature, Some(score))) + } + + private def diversityPenalty( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + ): Seq[Double] = { + val n = candidates.length + val selected = scala.collection.mutable.Set[Int]() + val newScores = Array.fill(n)(0.0) + val weight = query.params(CategoryDiversityRescoringWeightParam) + val k = query.params(CategoryDiversityKParam) + + val topKCategories = getCandidateCategory(k, candidates) + val clusterCntMap = scala.collection.mutable.Map[String, Int]() + + val candidatesWithIndexWithCategories: Seq[ + (CandidateWithFeatures[TweetCandidate], Int, Seq[String]) + ] = candidates.zipWithIndex.zip(topKCategories).map { + case ((candidate, index), categories) => + (candidate, index, categories) + } + for (i <- 0 until n) { + var maxScore = Double.NegativeInfinity + var bestCandidateIndex = -1 + var candidateCategories: Seq[String] = Seq.empty + + for ((candidate, index, categories) <- candidatesWithIndexWithCategories) { + if (!selected.contains(index)) { + val relevance = candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0) + var penalty = 0.0 + var totalCategoryCnt = 0 + + categories.foreach { category => + val cnt = clusterCntMap.getOrElse(category, 0) + penalty += math.log(cnt + 1) / math.log(2) + } + + val score = math.max(relevance - weight * penalty, 0.00001) + + if (score > maxScore) { + maxScore = score + bestCandidateIndex = index + candidateCategories = categories + } + } + } + if (bestCandidateIndex != -1) { + selected += bestCandidateIndex + newScores(bestCandidateIndex) = maxScore + candidateCategories.foreach { category => + clusterCntMap.put(category, clusterCntMap.getOrElse(category, 0) + 1) + } + } + } + + newScores.toSeq + } + + private def getCandidateCategory( + k: Int, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[Seq[String]] = { + val topKClusters: Seq[Seq[String]] = candidates.map { candidate => + val topKClustersMap = + Option( + candidate.features + .getOrElse(SimClustersLogFavBasedTweetFeature, new DataRecord()) + .getSparseContinuousFeatures + ).flatMap(mapOpt => + Option(mapOpt.get(SimclustersSparseTweetEmbeddingsFeature.getFeatureId))) + topKClustersMap + .map(_.asScala.toSeq.sortBy(-_._2).take(k).map(_._1)) + .getOrElse(Seq.empty) + } + topKClusters + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipEmbeddingFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipEmbeddingFeatureHydrator.scala new file mode 100644 index 000000000..82ded24c0 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipEmbeddingFeatureHydrator.scala @@ -0,0 +1,72 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.ClipEmbeddingFeaturesAdapter +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableClipEmbeddingFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableVideoClipEmbeddingFeatureHydrationDeciderParam +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.videoRecommendations.twitterClip.TwitterClipEmbeddingMhClientColumn +import com.twitter.util.logging.Logging + +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +@Singleton +class ClipEmbeddingFeatureHydrator @Inject() ( + twitterClipEmbeddingMhClientColumn: TwitterClipEmbeddingMhClientColumn, + statsReceiver: StatsReceiver) + extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] + with Logging { + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableClipEmbeddingFeaturesParam) && + query.params(EnableVideoClipEmbeddingFeatureHydrationDeciderParam) + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ClipEmbedding") + + private val fetcher: Fetcher[Long, Unit, Seq[Double]] = + twitterClipEmbeddingMhClientColumn.fetcher + + override val features: Set[Feature[_, _]] = Set(ClipEmbeddingFeature) + + private val DefaultFeatureMap = + FeatureMapBuilder().add(ClipEmbeddingFeature, new DataRecord()).build() + + private val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val clipEmbeddingNotFoundCounter = scopedStatsReceiver.counter("clipEmbeddingNotFound") + private val clipEmbeddingFoundCounter = scopedStatsReceiver.counter("clipEmbeddingFound") + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + existingFeatures: FeatureMap + ): Stitch[FeatureMap] = { + fetcher + .fetch(candidate.id).map { manhattanResult => + manhattanResult.v match { + case Some(embeddings) => + clipEmbeddingFoundCounter.incr() + val dataRecord = + ClipEmbeddingFeaturesAdapter.adaptToDataRecords(embeddings).asScala.head + FeatureMapBuilder().add(ClipEmbeddingFeature, dataRecord).build() + case None => + clipEmbeddingNotFoundCounter.incr() + DefaultFeatureMap + } + }.onFailure(e => { + error(s"Error fetching VideoClipEmbedding: $e") + }) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipEmbeddingMediaUnderstandingFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipEmbeddingMediaUnderstandingFeatureHydrator.scala new file mode 100644 index 000000000..7ae0504bc --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipEmbeddingMediaUnderstandingFeatureHydrator.scala @@ -0,0 +1,140 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.ClipEmbeddingFeaturesAdapter +import com.twitter.home_mixer.model.HomeFeatures.MediaCategoryFeature +import com.twitter.home_mixer.model.HomeFeatures.MediaIdFeature +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableClipEmbeddingMediaUnderstandingFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableVideoClipEmbeddingMediaUnderstandingFeatureHydrationDeciderParam +import com.twitter.media_understanding.embeddings.thriftscala.MediaEmbedding +import com.twitter.media_understanding.embeddings.thriftscala.MediaEmbeddingInfo +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Client +import com.twitter.strato.client.Fetcher +import com.twitter.util.logging.Logging +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object ClipEmbeddingFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class ClipEmbeddingMediaUnderstandingFeatureHydrator @Inject() ( + stratoClient: Client, + statsReceiver: StatsReceiver) + extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] + with Logging { + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableClipEmbeddingMediaUnderstandingFeaturesParam) && + query.params(EnableVideoClipEmbeddingMediaUnderstandingFeatureHydrationDeciderParam) + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ClipEmbeddingMediaUnderstanding") + + val MediaModelName: String = "twitter_clip_256" + val MediaVersion: String = "0" + + val mediaEmbeddingFetcher: Fetcher[(String, (String, String)), Unit, MediaEmbeddingInfo] = + stratoClient.fetcher[(String, (String, String)), Unit, MediaEmbeddingInfo]( + "media-understanding/embeddings/prod/embeddings" + ) + + def getTweetMediaEmbeddings(key: String): Stitch[Option[Seq[Double]]] = { + mediaEmbeddingFetchCounter.incr() + mediaEmbeddingFetcher + .fetch((key, (MediaModelName, MediaVersion))) + .map { result => + result.v match { + case Some(mediaData) => + mediaData.embedding match { + case Some(MediaEmbedding.DoubleVector(doubles)) => + mediaEmbeddingFetchSuccessCounter.incr() + Some(doubles.map(_.toDouble)) + case _ => + mediaEmbeddingFetchFailureCounter.incr() + None + } + case None => + mediaEmbeddingFetchFailureCounter.incr() + None + } + } + .rescue { + case e: Exception => + mediaEmbeddingFetchFailureCounter.incr() + Stitch.None + } + } + + def getClipEmbeddings( + tweetId: Long, + existingFeatures: FeatureMap + ): Stitch[Option[Seq[Double]]] = { + val mediaId = existingFeatures.getOrElse(MediaIdFeature, None) + val mediaCategory = existingFeatures.getOrElse(MediaCategoryFeature, None) + + (mediaId, mediaCategory) match { + case (Some(id), Some(category)) => + val key = s"${category.getValue}/$id" + getTweetMediaEmbeddings(key).map { + case Some(embeddings) => Some(embeddings) + case _ => + clipEmbeddingNotFoundNoneCounter.incr() + None + } + case _ => Stitch.value(None) + } + } + + override val features: Set[Feature[_, _]] = Set(ClipEmbeddingFeature) + + private val DefaultFeatureMap = + FeatureMapBuilder().add(ClipEmbeddingFeature, new DataRecord()).build() + + private val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val clipEmbeddingNotFoundNoneCounter = + scopedStatsReceiver.counter("clipEmbeddingNotFoundNoneCounter") + private val clipEmbeddingNotFoundCounter = scopedStatsReceiver.counter("clipEmbeddingNotFound") + private val clipEmbeddingFoundCounter = scopedStatsReceiver.counter("clipEmbeddingFound") + + val mediaEmbeddingFetchCounter = statsReceiver.counter("media_embedding_fetch_total") + val mediaEmbeddingFetchSuccessCounter = statsReceiver.counter("media_embedding_fetch_success") + val mediaEmbeddingFetchFailureCounter = statsReceiver.counter("media_embedding_fetch_failure") + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + existingFeatures: FeatureMap + ): Stitch[FeatureMap] = { + getClipEmbeddings(candidate.id, existingFeatures) + .map { + case Some(embeddings) => + clipEmbeddingFoundCounter.incr() + val dataRecord = + ClipEmbeddingFeaturesAdapter.adaptToDataRecords(embeddings).asScala.head + FeatureMapBuilder().add(ClipEmbeddingFeature, dataRecord).build() + case None => + clipEmbeddingNotFoundCounter.incr() + DefaultFeatureMap + }.onFailure { + case e: Exception => error(s"Error fetching VideoClipEmbedding: $e") + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipImageClusterIdFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipImageClusterIdFeatureHydrator.scala new file mode 100644 index 000000000..66d00759a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipImageClusterIdFeatureHydrator.scala @@ -0,0 +1,95 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.ClipImageClusterIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.ImageClipClusterIdInMemCache +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.videoRecommendations.twitterClip.TwitterClipImageClusterIdMh95ClientColumn +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class ClipImageClusterIdFeatureHydrator @Inject() ( + twitterClipImageClusterIdMh95ClientColumn: TwitterClipImageClusterIdMh95ClientColumn, + @Named(ImageClipClusterIdInMemCache) imageClipClusterIdInMemCache: InProcessCache[ + Long, + Option[Option[Long]] + ], + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ClipImageClusterId") + + override val features: Set[Feature[_, _]] = Set(ClipImageClusterIdsFeature) + + private val clusterIdFetcher = twitterClipImageClusterIdMh95ClientColumn.fetcher + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") + private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") + private val fetchExceptionCounter = + scopedStatsReceiver.counter("getFromCacheOrFetch/exception") + + private def getFromCacheOrFetch(mediaId: Long): Stitch[Option[Option[Long]]] = { + imageClipClusterIdInMemCache + .get(mediaId) + .map { cachedValue => + cacheHitCounter.incr() + Stitch.value(cachedValue) + }.getOrElse { + cacheMissCounter.incr() + clusterIdFetcher + .fetch(mediaId) + .map(_.v) + .liftToOption() + .flatMap { clusterIdOpt => + imageClipClusterIdInMemCache.set(mediaId, clusterIdOpt) + Stitch.value(clusterIdOpt) + } + }.handle { + case _: Exception => + fetchExceptionCounter.incr() + None + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + Stitch.collect( + candidates.map { candidate => + val mediaIds = candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty[Long]) + val items: Seq[Stitch[Option[(Long, Long)]]] = mediaIds.map { mediaId => + getFromCacheOrFetch(mediaId).map { + case Some(Some(mediaClusterId)) => + keyFoundCounter.incr() + Some(mediaId -> mediaClusterId) + case _ => + keyNotFoundCounter.incr() + None + } + } + + Stitch.collect(items).map { results => + val mediaClusterMap = results.flatten.toMap + FeatureMap(ClipImageClusterIdsFeature, mediaClusterMap) + } + } + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DependentBulkCandidateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DependentBulkCandidateFeatureHydrator.scala new file mode 100644 index 000000000..3687b2fd7 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DependentBulkCandidateFeatureHydrator.scala @@ -0,0 +1,77 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch + +case class DependentBulkCandidateFeatureHydrator[Query <: PipelineQuery]( + parentCandidateFeatureHydrator: BulkCandidateFeatureHydrator[Query, TweetCandidate], + childrenCandidateFeatureHydrators: Seq[ + BulkCandidateFeatureHydrator[Query, TweetCandidate] with WithDefaultFeatureMap + ]) extends BulkCandidateFeatureHydrator[Query, TweetCandidate] + with Conditionally[Query] { + + override final val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "Dependent" + + parentCandidateFeatureHydrator.identifier.name) + + override def onlyIf(query: Query): Boolean = { + parentCandidateFeatureHydrator match { + case candidateHydrator: BulkCandidateFeatureHydrator[_, _] with Conditionally[Query] => + candidateHydrator.onlyIf(query) + case _ => true + } + } + + override val features: Set[Feature[_, _]] = + childrenCandidateFeatureHydrators.foldLeft(parentCandidateFeatureHydrator.features) { + (features, childFeatureHydrator) => features ++ childFeatureHydrator.features + } + + override def apply( + query: Query, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = + OffloadFuturePools.offloadStitch { + val parentFeatureMapsStitch = parentCandidateFeatureHydrator.apply(query, candidates) + parentFeatureMapsStitch.flatMap { parentFeatureMaps => + val updatedCandidates = candidates.zip(parentFeatureMaps).map { + case (tweetCandidate, featureMap) => + new CandidateWithFeatures[TweetCandidate] { + override val candidate = tweetCandidate.candidate + override val features = tweetCandidate.features ++ featureMap + } + } + val (childrenCandidateFeatureHydratorsFiltered, childrenCandidateFeatureHydratorsDefault) = + childrenCandidateFeatureHydrators.partition { + case candidateFeatureHydrator: BulkCandidateFeatureHydrator[_, _] with Conditionally[ + Query + ] => + candidateFeatureHydrator.onlyIf(query) + + case _: BulkCandidateFeatureHydrator[_, _] => + true + + case _ => false + } + val childrenFeatureMapsDefault = childrenCandidateFeatureHydratorsDefault.map { + featureHydrator => Seq.fill(candidates.size)(featureHydrator.defaultFeatureMap) + } + val childrenFeatureMapsStitch = Stitch.traverse(childrenCandidateFeatureHydratorsFiltered) { + _.apply(query, updatedCandidates) + } + childrenFeatureMapsStitch.map { childrenFeatureMaps => + val allFeatureMaps = + (parentFeatureMaps +: childrenFeatureMaps) ++ childrenFeatureMapsDefault + allFeatureMaps.transpose.map(FeatureMap.merge(_)) + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DiversityRescoringFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DiversityRescoringFeatureHydrator.scala new file mode 100644 index 000000000..ee2d5c47f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DiversityRescoringFeatureHydrator.scala @@ -0,0 +1,119 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import breeze.linalg._ +import breeze.numerics.sqrt +//import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinEmbeddingsFeatures.TwhinTweetEmbeddingsFeature +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.TransformerEmbeddingsFeatures.PostTransformerEmbeddingsJointBlueFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.TwhinDiversityRescoringWeightParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.TwhinDiversityRescoringRatioParam +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import scala.jdk.CollectionConverters.collectionAsScalaIterableConverter + +object DiversityRescoringFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("DiversityRescoring") + + override val features: Set[Feature[_, _]] = Set(ScoreFeature) + + final val EmptyDataRecord = new DataRecord() + + private val embeddingsSize = 128 + + private val defaultEmbeddings = Seq.fill(embeddingsSize)(0.0) + + private val defaultDenseVector = + DenseVector(Array.fill(embeddingsSize)(1.0)) / sqrt(embeddingsSize) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { + val embeddings = candidates.map { candidate => + val embeddingsTensor = + Option( + candidate.features + .getOrElse(TransformerPostEmbeddingJointBlueFeature, EmptyDataRecord) + .getTensors) + .flatMap(tensorsOpt => + Option(tensorsOpt.get(PostTransformerEmbeddingsJointBlueFeature.getFeatureId))) + embeddingsTensor + .map(_.getFloatTensor.floats.asScala.map(_.doubleValue).toSeq) + .getOrElse(defaultEmbeddings) + } + + val denseEmbeddingsNormalized = embeddings.map { seq => + val denseVector = DenseVector(seq.toArray) + val normVal = norm(denseVector) + if (normVal != 0) + denseVector / normVal + else defaultDenseVector + } + + val distanceMatrix = + DenseMatrix.zeros[Double](denseEmbeddingsNormalized.length, denseEmbeddingsNormalized.length) + + for (i <- denseEmbeddingsNormalized.indices; j <- denseEmbeddingsNormalized.indices) { + distanceMatrix(i, j) = norm(denseEmbeddingsNormalized(i) - denseEmbeddingsNormalized(j)) + } + mmr(query, candidates, distanceMatrix).map(score => FeatureMap(ScoreFeature, Some(score))) + } + + def mmr( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + distanceMatrix: DenseMatrix[Double] + ): Seq[Double] = { + val n = candidates.length + val diversityRatio = query.params(TwhinDiversityRescoringRatioParam) + val diversityWeight = query.params(TwhinDiversityRescoringWeightParam) + val selected = scala.collection.mutable.Set[Int]() + val newScores = Array.fill(n)(0.0) + + val candidatesWithIndex = candidates.zipWithIndex + for (i <- 0 until n) { + var maxScore = Double.NegativeInfinity + var bestCandidateIndex = -1 + + for ((candidate, index) <- candidatesWithIndex) { + if (!selected.contains(index)) { + val relevance = candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0) + val minDistance = { + if (selected.isEmpty || selected.size < (1 - diversityRatio) * n) None + else { + selected + .map(j => distanceMatrix(index, j)) + .reduceOption { (a, b) => + if (a < b) a else b + } + } + } + val score = relevance + diversityWeight * minDistance.getOrElse(2.0) + if (score > maxScore) { + maxScore = score + bestCandidateIndex = index + } + } + } + + if (bestCandidateIndex != -1) { + selected += bestCandidateIndex + newScores(bestCandidateIndex) = maxScore + } + } + + newScores.toSeq + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/EarlybirdSearchResultFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/EarlybirdSearchResultFeatureHydrator.scala new file mode 100644 index 000000000..6ddc2fb64 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/EarlybirdSearchResultFeatureHydrator.scala @@ -0,0 +1,75 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository +import com.twitter.home_mixer.util.ObservedKeyValueResultHandler +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.servo.keyvalue.KeyValueResult +import com.twitter.servo.repository.KeyValueRepository +import com.twitter.stitch.Stitch +import com.twitter.util.Return +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class EarlybirdSearchResultFeatureHydrator @Inject() ( + @Named(EarlybirdRepository) client: KeyValueRepository[ + (Seq[Long], Long), + Long, + eb.ThriftSearchResult + ], + override val statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with ObservedKeyValueResultHandler { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("EarlybirdSearchResult") + + override val features: Set[Feature[_, _]] = Set( + EarlybirdScoreFeature, + EarlybirdSearchResultFeature + ) + + override val statScope: String = identifier.toString + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + client((candidates.map(_.candidate.id), query.getRequiredUserId)) + .map(handleResponse(candidates, _)) + } + + private def handleResponse( + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + results: KeyValueResult[Long, eb.ThriftSearchResult] + ): Seq[FeatureMap] = { + candidates + .map { candidate => + observedGet(Some(candidate.candidate.id), results) + }.map { + case Return(Some(searchResult)) => + FeatureMapBuilder() + .add(EarlybirdScoreFeature, searchResult.metadata.flatMap(_.score)) + .add(EarlybirdSearchResultFeature, Some(searchResult)) + .build() + case other => + FeatureMapBuilder() + .add(EarlybirdScoreFeature, None) + .add(EarlybirdSearchResultFeature, other) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala index c0793be4b..fa6294a7d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala @@ -1,6 +1,8 @@ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature +import com.twitter.home_mixer.model.HomeFeatures.HasRecentFeedbackSinceCacheTtlFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder @@ -9,6 +11,7 @@ import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIde import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClient +import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton @@ -19,7 +22,8 @@ case class FeedbackHistoryQueryFeatureHydrator @Inject() ( override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory") - override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature) + override val features: Set[Feature[_, _]] = + Set(FeedbackHistoryFeature, HasRecentFeedbackSinceCacheTtlFeature) override def hydrate( query: PipelineQuery @@ -27,6 +31,20 @@ case class FeedbackHistoryQueryFeatureHydrator @Inject() ( Stitch .callFuture(feedbackHistoryClient.get(query.getRequiredUserId)) .map { feedbackHistory => - FeatureMapBuilder().add(FeedbackHistoryFeature, feedbackHistory).build() + val latestFeedbackTimestamp = + if (feedbackHistory.nonEmpty) Some(feedbackHistory.map(_.timestamp.inMilliseconds).max) + else None + val cachedScoredTweetsTtl = query.params(CachedScoredTweets.TTLParam) + val hasRecentFeedbackSinceCacheTtl = latestFeedbackTimestamp match { + case Some(timestamp) => + Time.fromMilliseconds(timestamp).untilNow < cachedScoredTweetsTtl + case None => false + } + + val featureMapBuilder = FeatureMapBuilder() + .add(FeedbackHistoryFeature, feedbackHistory) + .add(HasRecentFeedbackSinceCacheTtlFeature, hasRecentFeedbackSinceCacheTtl) + + featureMapBuilder.build() } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowableUttTopicsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowableUttTopicsQueryFeatureHydrator.scala new file mode 100644 index 000000000..07a97a4de --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowableUttTopicsQueryFeatureHydrator.scala @@ -0,0 +1,53 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.util.ObservedKeyValueResultHandler +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.topic_recos.common.LocaleUtil +import com.twitter.topiclisting.SemanticCoreEntityId +import com.twitter.topiclisting.TopicListingViewerContext +import com.twitter.tsp.stores.UttTopicFilterStore +import com.twitter.tsp.{thriftscala => tsp} +import javax.inject.Inject +import javax.inject.Singleton + +object FollowableUttTopicsFeatures + extends Feature[PipelineQuery, Option[Map[SemanticCoreEntityId, Option[tsp.TopicFollowType]]]] + +@Singleton +class FollowableUttTopicsQueryFeatureHydrator @Inject() ( + uttStore: UttTopicFilterStore, + override val statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with ObservedKeyValueResultHandler { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("FollowableUttTopics") + + override val features: Set[Feature[_, _]] = Set(FollowableUttTopicsFeatures) + + override val statScope: String = identifier.toString + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val context = TopicListingViewerContext.fromClientContext(query.clientContext) + + Stitch.callFuture { + uttStore + .getAllowListTopicsForUser( + userId = query.getRequiredUserId, + topicListingSetting = tsp.TopicListingSetting.Followable, + context = context + .copy(languageCode = LocaleUtil.getStandardLanguageCode(context.languageCode)), + bypassModes = None + ).map { topics => + FeatureMapBuilder().add(FollowableUttTopicsFeatures, Some(topics)).build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala new file mode 100644 index 000000000..3f2b1effc --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala @@ -0,0 +1,66 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.follow_recommendations.{thriftscala => frs} +import com.twitter.product_mixer.component_library.candidate_source.recommendations.UserFollowRecommendationsCandidateSource +import com.twitter.product_mixer.component_library.candidate_source.recommendations.CachedUserFollowRecommendationsCandidateSource +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +object FrsSeedUserIdsFeature extends Feature[TweetCandidate, Option[Seq[Long]]] +object FrsUserToFollowedByUserIdsFeature extends Feature[TweetCandidate, Map[Long, Seq[Long]]] + +@Singleton +case class FrsSeedUsersQueryFeatureHydrator @Inject() ( + userFollowRecommendationsCandidateSource: UserFollowRecommendationsCandidateSource, + cachedUserFollowRecommendationsCandidateSource: CachedUserFollowRecommendationsCandidateSource) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FrsSeedUsers") + + private val maxUsersToFetch = 100 + + override val features: Set[Feature[_, _]] = Set( + FrsSeedUserIdsFeature, + FrsUserToFollowedByUserIdsFeature + ) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val frsRequest = frs.RecommendationRequest( + clientContext = frs.ClientContext(query.getOptionalUserId), + displayLocation = frs.DisplayLocation.HomeTimelineTweetRecs, + maxResults = Some(maxUsersToFetch) + ) + + userFollowRecommendationsCandidateSource(StratoKeyView(frsRequest, Unit)) + .map { userRecommendations: Seq[frs.UserRecommendation] => + val seedUserIds = userRecommendations.map(_.userId) + val seedUserIdsSet = seedUserIds.toSet + + val userToFollowedByUserIds: Map[Long, Seq[Long]] = userRecommendations.flatMap { + userRecommendation => + if (seedUserIdsSet.contains(userRecommendation.userId)) { + val followProof = + userRecommendation.reason.flatMap(_.accountProof).flatMap(_.followProof) + val followedByUserIds = followProof.map(_.userIds).getOrElse(Seq.empty) + Some(userRecommendation.userId -> followedByUserIds) + } else { + None + } + }.toMap + + FeatureMapBuilder() + .add(FrsSeedUserIdsFeature, Some(seedUserIds)) + .add(FrsUserToFollowedByUserIdsFeature, userToFollowedByUserIds) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GeoduckAuthorLocationHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GeoduckAuthorLocationHydrator.scala new file mode 100644 index 000000000..aafb8509f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GeoduckAuthorLocationHydrator.scala @@ -0,0 +1,200 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.dal.personal_data.{thriftjava => pd} +import com.twitter.geoduck.common.thriftscala.TransactionLocation +import com.twitter.geoduck.common.{thriftscala => t} +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.product_mixer.component_library.feature.location.Location +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature +import com.twitter.product_mixer.core.feature.datarecord.LongDiscreteDataRecordCompatible +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ExpiringLruInProcessCache +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.geo.service.UserLocationClientColumn +import com.twitter.timelines.prediction.features.location.LocationFeatures +import javax.inject.Inject +import javax.inject.Singleton + +object AuthorLocationNeighborhoodFeature + extends DataRecordOptionalFeature[PipelineQuery, Long] + with LongDiscreteDataRecordCompatible { + override val featureName: String = + LocationFeatures.AUTHOR_LOCATION_NEIGHBORDHOOD.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredLocation) +} + +object AuthorLocationCityFeature + extends DataRecordOptionalFeature[PipelineQuery, Long] + with LongDiscreteDataRecordCompatible { + override val featureName: String = + LocationFeatures.AUTHOR_LOCATION_CITY.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredLocation) +} + +object AuthorLocationMetroFeature + extends DataRecordOptionalFeature[PipelineQuery, Long] + with LongDiscreteDataRecordCompatible { + override val featureName: String = + LocationFeatures.AUTHOR_LOCATION_METRO.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredLocation) +} + +object AuthorLocationRegionFeature + extends DataRecordOptionalFeature[PipelineQuery, Long] + with LongDiscreteDataRecordCompatible { + override val featureName: String = + LocationFeatures.AUTHOR_LOCATION_REGION.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredLocation) +} + +object AuthorLocationCountryFeature + extends DataRecordOptionalFeature[PipelineQuery, Long] + with LongDiscreteDataRecordCompatible { + override val featureName: String = + LocationFeatures.AUTHOR_LOCATION_COUNTRY.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredLocation) +} + +object GeoduckAuthorLocationHydrator { + private val BaseTTLMinutes = 60 * 24 + private val TTL = (BaseTTLMinutes + scala.util.Random.nextInt(60)).minutes + + val cache: InProcessCache[Long, Option[TransactionLocation]] = + new ExpiringLruInProcessCache[Long, Option[TransactionLocation]]( + ttl = TTL, + maximumSize = 150 * 1000 // Cache up to 150k users + ) +} + +@Singleton +class GeoduckAuthorLocationHydrator @Inject() ( + userLocationClientColumn: UserLocationClientColumn) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "GeoduckAuthorLocationHydrator") + + override val features: Set[Feature[_, _]] = Set( + LocationFeature, + AuthorLocationNeighborhoodFeature, + AuthorLocationCityFeature, + AuthorLocationMetroFeature, + AuthorLocationRegionFeature, + AuthorLocationCountryFeature + ) + + private val PlaceQuery = t.PlaceQuery( + placeTypes = Some( + Set( + t.PlaceType.Neighborhood, + t.PlaceType.City, + t.PlaceType.Metro, + t.PlaceType.Admin1, + t.PlaceType.Country + ) + ) + ) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct + + val (hitIds, missIds) = authorIds.partition(id => + GeoduckAuthorLocationHydrator.cache + .get(id) + .isDefined) + + val cachedOptionMap: Map[Long, Option[TransactionLocation]] = + hitIds.map(id => id -> GeoduckAuthorLocationHydrator.cache.get(id).get).toMap + + val fetchCacheMisses: Stitch[Map[Long, TransactionLocation]] = + if (missIds.isEmpty) Stitch.value(Map.empty[Long, TransactionLocation]) + else { + userLocationClientColumn.fetcher + .fetch( + key = Unit, + t.UserLocationRequest( + userIds = missIds, + placeQuery = Some(PlaceQuery) + ) + ) + .map { response => + response.v.toList.flatMap(_._1).toMap + } + .handle { case _ => Map.empty[Long, TransactionLocation] } + } + + val allLocationsStitch: Stitch[Map[Long, TransactionLocation]] = + fetchCacheMisses.map { fetchedMap => + missIds.foreach { id => + val locOpt: Option[TransactionLocation] = fetchedMap.get(id) + GeoduckAuthorLocationHydrator.cache.set(id, locOpt) + } + + val cachedLocs: Map[Long, TransactionLocation] = + cachedOptionMap.collect { case (id, Some(loc)) => id -> loc } + + cachedLocs ++ fetchedMap + } + + allLocationsStitch.map { allLocations => + candidates.map { candidate => + val locOpt = for { + authorId <- candidate.features.getOrElse(AuthorIdFeature, None) + loc <- allLocations.get(authorId) + } yield loc + + locOpt + .map { transactionLocation => + val placeMap = transactionLocation.placeMap + val locationDetails = Location( + neighborhood = placeMap + .flatMap(_.get(t.PlaceType.Neighborhood)) + .flatMap(_.headOption), + city = placeMap.flatMap(_.get(t.PlaceType.City)).flatMap(_.headOption), + metro = placeMap.flatMap(_.get(t.PlaceType.Metro)).flatMap(_.headOption), + region = placeMap.flatMap(_.get(t.PlaceType.Admin1)).flatMap(_.headOption), + country = placeMap.flatMap(_.get(t.PlaceType.Country)).flatMap(_.headOption) + ) + + FeatureMapBuilder() + .add(LocationFeature, Option(locationDetails)) + .add(AuthorLocationNeighborhoodFeature, locationDetails.neighborhood) + .add(AuthorLocationCityFeature, locationDetails.city) + .add(AuthorLocationMetroFeature, locationDetails.metro) + .add(AuthorLocationRegionFeature, locationDetails.region) + .add(AuthorLocationCountryFeature, locationDetails.country) + .build() + } + .getOrElse { + FeatureMapBuilder() + .add(LocationFeature, None) + .add(AuthorLocationNeighborhoodFeature, None) + .add(AuthorLocationCityFeature, None) + .add(AuthorLocationMetroFeature, None) + .add(AuthorLocationRegionFeature, None) + .add(AuthorLocationCountryFeature, None) + .build() + } + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala new file mode 100644 index 000000000..18efc75d0 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala @@ -0,0 +1,209 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.github.nscala_time.time.Imports.LocalDate +import com.twitter.ads.entities.db.{thriftscala => ae} +import com.twitter.conversions.DurationOps._ +import com.twitter.gizmoduck.{thriftscala => gt} +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.gizmoduck_features.GFeatures +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.gizmoduck_features.GizmoduckFeaturesAdapter +import com.twitter.home_mixer.model.HomeFeatures.AuthorAccountAge +import com.twitter.home_mixer.model.HomeFeatures.AuthorFollowersFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsProtectedFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.IsSupportAccountReplyFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorSafetyLabels +import com.twitter.home_mixer.module.SupportAccountsConfig +import com.twitter.home_mixer.param.HomeMixerInjectionNames.GizmoduckTimelinesCache +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.TtlCache +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.stitch.Stitch +import com.twitter.util.Duration +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +private case class UserFeatures( + isBlueVerified: Boolean, + isVerifiedOrganization: Boolean, + isVerifiedOrganizationAffiliate: Boolean, + isProtected: Boolean, + isSupportAccount: Boolean, + followersCount: Option[Long], + accountAge: Option[Duration], + labels: Option[Seq[String]]) + extends GFeatures + +object GizmoduckAuthorFeatures + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class GizmoduckAuthorFeatureHydrator @Inject() ( + gizmoduck: gt.UserService.MethodPerEndpoint, + @Named(GizmoduckTimelinesCache) cacheClient: TtlCache[Long, gt.User], + supportAccounts: SupportAccountsConfig) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GizmoduckAuthor") + + override val features: Set[Feature[_, _]] = Set( + AuthorAccountAge, + AuthorFollowersFeature, + AuthorIsBlueVerifiedFeature, + AuthorIsProtectedFeature, + IsSupportAccountReplyFeature, + GizmoduckAuthorFeatures, + AuthorSafetyLabels + ) + + private val CacheTTL = 24.hours + + private val queryFields: Set[gt.QueryFields] = Set( + gt.QueryFields.AdvertiserAccount, + gt.QueryFields.Profile, + gt.QueryFields.Safety, + gt.QueryFields.Labels, + gt.QueryFields.Counts + ) + + private val lookupContext = gt.LookupContext(isRequestSheddable = Some(true)) + + // Advertiser service levels which are assumed to provide support via replies + private val AdvertiserServiceLevels = Set[ae.ServiceLevel]( + ae.ServiceLevel.Dso, + ae.ServiceLevel.Mms, + ae.ServiceLevel.Reseller, + ae.ServiceLevel.Smb + ) + + private val SupportAccounts = supportAccounts.accounts + + private val DefaultUserFeatures = UserFeatures( + isBlueVerified = false, + isVerifiedOrganization = false, + isVerifiedOrganizationAffiliate = false, + isProtected = false, + isSupportAccount = false, + followersCount = None, + accountAge = None, + labels = None + ) + + private def extractFeaturesFromUser( + user: gt.User + ): UserFeatures = { + val isBlueVerified = user.safety.flatMap(_.isBlueVerified).getOrElse(false) + val safetyLabels = user.labels.map(_.labels.map(_.labelValue.name)) + val verifiedOrganizationDetails = user.safety.flatMap(_.verifiedOrganizationDetails) + val isVerifiedOrganization = + verifiedOrganizationDetails.flatMap(_.isVerifiedOrganization).getOrElse(false) + val isVerifiedOrganizationAffiliate = + verifiedOrganizationDetails.flatMap(_.isVerifiedOrganizationAffiliate).getOrElse(false) + + val isProtected = user.safety.exists(_.isProtected) + val isSupportAccount = + user.profile.exists(_.businessProfileState == gt.BusinessProfileState.Enabled) || + (user.advertiserAccount.flatMap(_.advertiserType).nonEmpty && + user.advertiserAccount + .flatMap(_.serviceLevels).getOrElse(Seq.empty) + .exists(AdvertiserServiceLevels.contains)) + val followersCount = user.counts.map(_.followers) + val accountAge = Duration.fromSeconds( + (LocalDate.now().toDate.toInstant.getEpochSecond - (user.createdAtMsec / 1000)).toInt) + + UserFeatures( + isBlueVerified, + isVerifiedOrganization, + isVerifiedOrganizationAffiliate, + isProtected, + isSupportAccount, + followersCount, + Some(accountAge), + safetyLabels + ) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct + cacheClient + .get(authorIds) + .flatMap { cacheResponse => + val cacheHydratedUsersMap = cacheResponse.found + val notFoundUsers = cacheResponse.notFound.toSeq.distinct + + val gizmoduckResponseFuture = if (notFoundUsers.nonEmpty) { + gizmoduck.get(lookupContext, notFoundUsers, queryFields) + } else Future.value(Seq.empty) + + gizmoduckResponseFuture.map { gizmoduckResponse => + val gizmoduckHydratedUsersMap = gizmoduckResponse.collect { + case userResult if userResult.user.isDefined => + val user = userResult.user.get + cacheClient.add(user.id, user, CacheTTL) + user.id -> user + }.toMap + + val hydratedUsersMap = (cacheHydratedUsersMap ++ gizmoduckHydratedUsersMap) + .mapValues(extractFeaturesFromUser) + + candidates.map { candidate => + val authorIdOpt = candidate.features.getOrElse(AuthorIdFeature, None) + val userFeatures = + authorIdOpt.flatMap(hydratedUsersMap.get).getOrElse(DefaultUserFeatures) + + // Some accounts run promotions and send replies automatically. + // We assume that a reply that took more than one minute is not an auto-reply. + // If time difference doesn't exist, this means that one of the tweets was + // not snowflake and therefore much older, and therefore OK as an extended reply. + val timeDifference = candidate.features.getOrElse(InReplyToTweetIdFeature, None).map { + SnowflakeId.timeFromId(candidate.candidate.id) - SnowflakeId.timeFromId(_) + } + val isAutoReply = timeDifference.exists(_ < 1.minute) + + val isSupportAccountReply = + candidate.features.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && + !candidate.features.getOrElse(IsRetweetFeature, false) && + candidate.features.getOrElse(FromInNetworkSourceFeature, false) && + (userFeatures.isSupportAccount || + authorIdOpt.exists(SupportAccounts.contains) || isAutoReply) + + val gizmoduckFeaturesRichDataRecords = + GizmoduckFeaturesAdapter.adaptToDataRecords(userFeatures).asScala.head + + FeatureMapBuilder() + .add(AuthorAccountAge, userFeatures.accountAge) + .add(AuthorFollowersFeature, userFeatures.followersCount) + .add(AuthorIsBlueVerifiedFeature, userFeatures.isBlueVerified) + .add(AuthorIsProtectedFeature, userFeatures.isProtected) + .add(IsSupportAccountReplyFeature, isSupportAccountReply) + .add(GizmoduckAuthorFeatures, gizmoduckFeaturesRichDataRecords) + .add(AuthorSafetyLabels, userFeatures.labels) + .build() + } + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.scala index 431711900..07a7d58d4 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.scala @@ -1,9 +1,19 @@ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.gizmoduck.{thriftscala => gt} +import com.twitter.home_mixer.model.HomeFeatures.SignupCountryFeature +import com.twitter.home_mixer.model.HomeFeatures.SignupSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.UserFollowersCountFeature import com.twitter.home_mixer.model.HomeFeatures.UserFollowingCountFeature import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.UserTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsAdsPersonalizationFeature +import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsDataSharingFeature +import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsForYouRecommendationsFeature +import com.twitter.home_mixer.model.HomeFeatures.ViewerHasPremiumTier +import com.twitter.home_mixer.model.HomeFeatures.ViewerSafetyLabels +import com.twitter.home_mixer.model.signup.MarchMadness +import com.twitter.home_mixer.model.signup.Onboard import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -13,6 +23,7 @@ import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIde import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.stitch.gizmoduck.Gizmoduck + import javax.inject.Inject import javax.inject.Singleton @@ -22,11 +33,27 @@ case class GizmoduckUserQueryFeatureHydrator @Inject() (gizmoduck: Gizmoduck) override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GizmoduckUser") - override val features: Set[Feature[_, _]] = - Set(UserFollowingCountFeature, UserTypeFeature, UserScreenNameFeature) + override val features: Set[Feature[_, _]] = Set( + UserFollowingCountFeature, + UserFollowersCountFeature, + UserTypeFeature, + UserScreenNameFeature, + ViewerHasPremiumTier, + SignupCountryFeature, + SignupSourceFeature, + ViewerAllowsForYouRecommendationsFeature, + ViewerAllowsDataSharingFeature, + ViewerAllowsAdsPersonalizationFeature, + ViewerSafetyLabels + ) - private val queryFields: Set[gt.QueryFields] = - Set(gt.QueryFields.Counts, gt.QueryFields.Safety, gt.QueryFields.Profile) + private val queryFields: Set[gt.QueryFields] = Set( + gt.QueryFields.Counts, + gt.QueryFields.Safety, + gt.QueryFields.Profile, + gt.QueryFields.Account, + gt.QueryFields.Labels + ) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId @@ -36,10 +63,45 @@ case class GizmoduckUserQueryFeatureHydrator @Inject() (gizmoduck: Gizmoduck) queryFields = queryFields, context = gt.LookupContext(forUserId = Some(userId), includeSoftUsers = true)) .map { user => + val premiumTier = user.safety + .map { safety => + safety.skipRateLimit.contains(true) || + safety.isBlueVerified.contains(true) || + safety.verifiedType.contains(gt.VerifiedType.Business) || + safety.verifiedType.contains(gt.VerifiedType.Government) || + safety.verifiedOrganizationDetails.exists(_.isVerifiedOrganization.getOrElse(false)) || + safety.verifiedOrganizationDetails + .exists(_.isVerifiedOrganizationAffiliate.contains(true)) + }.getOrElse(false) + + val signupSource = user.safety.flatMap(_.signupCreationSource).flatMap { + case gt.SignupCreationSource.MarchMadness => Some(MarchMadness) + case gt.SignupCreationSource.Onboard => Some(Onboard) + case _ => None + } + FeatureMapBuilder() .add(UserFollowingCountFeature, user.counts.map(_.following.toInt)) + .add(UserFollowersCountFeature, user.counts.map(_.followers.toInt)) .add(UserTypeFeature, Some(user.userType)) .add(UserScreenNameFeature, user.profile.map(_.screenName)) + .add(ViewerHasPremiumTier, premiumTier) + .add(SignupCountryFeature, user.safety.flatMap(_.signupCountryCode)) + .add(SignupSourceFeature, signupSource) + .add( + ViewerAllowsForYouRecommendationsFeature, + user.account.flatMap(_.allowForYouRecommendations) + ) + .add( + ViewerAllowsDataSharingFeature, + user.account.map(_.allowSharingDataForThirdPartyPersonalization) + ).add( + ViewerAllowsAdsPersonalizationFeature, + user.account.map(_.allowAdsPersonalization) + ).add( + ViewerSafetyLabels, + user.labels.map(_.labels.map(_.labelValue.name)) + ) .build() } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GraphTwoHopFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GraphTwoHopFeatureHydrator.scala new file mode 100644 index 000000000..7583ba53e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GraphTwoHopFeatureHydrator.scala @@ -0,0 +1,101 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.graph_feature_service.{thriftscala => gfs} +import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer.util.ObservedKeyValueResultHandler +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.repository.KeyValueRepository +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.adapters.two_hop_features.TwoHopFeaturesAdapter +import com.twitter.util.Try +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object GraphTwoHopFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class GraphTwoHopFeatureHydrator @Inject() ( + @Named(GraphTwoHopRepository) client: KeyValueRepository[ + (Seq[Long], Long), + Long, + Seq[gfs.IntersectionValue] + ], + override val statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with ObservedKeyValueResultHandler { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GraphTwoHop") + + override val features: Set[Feature[_, _]] = Set(GraphTwoHopFeature, FollowedByUserIdsFeature) + + override val statScope: String = identifier.toString + + private val twoHopFeaturesAdapter = new TwoHopFeaturesAdapter + + private val FollowFeatureType = gfs.FeatureType(gfs.EdgeType.Following, gfs.EdgeType.FollowedBy) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + // Apply filters to in network candidates for retweets only. + val (inNetworkCandidates, oonCandidates) = candidates.partition { candidate => + candidate.features.getOrElse(FromInNetworkSourceFeature, false) + } + + val inNetworkCandidatesToHydrate = + inNetworkCandidates.filter(_.features.getOrElse(IsRetweetFeature, false)) + + val candidatesToHydrate = (inNetworkCandidatesToHydrate ++ oonCandidates) + .flatMap(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)).distinct + + val response = client((candidatesToHydrate, query.getRequiredUserId)) + + response.map { result => + candidates.map { candidate => + val originalAuthorId = CandidatesUtil.getOriginalAuthorId(candidate.features) + + val value = observedGet(key = originalAuthorId, keyValueResult = result) + val transformedValue = postTransformer(value) + val followedByUserIds = value.toOption + .flatMap(getFollowedByUserIds(_)) + .getOrElse(Seq.empty) + + FeatureMapBuilder() + .add(GraphTwoHopFeature, transformedValue) + .add(FollowedByUserIdsFeature, followedByUserIds) + .build() + } + } + } + + private def getFollowedByUserIds(input: Option[Seq[gfs.IntersectionValue]]): Option[Seq[Long]] = + input.map { + _.filter(_.featureType == FollowFeatureType).flatMap(_.intersectionIds).flatten.distinct + } + + private def postTransformer(input: Try[Option[Seq[gfs.IntersectionValue]]]): Try[DataRecord] = + input.map(twoHopFeaturesAdapter.adaptToDataRecords(_).asScala.head) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokAnnotationsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokAnnotationsFeatureHydrator.scala new file mode 100644 index 000000000..21a154bee --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokAnnotationsFeatureHydrator.scala @@ -0,0 +1,206 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.GrokTopics.GrokCategoryIdToNameMap +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokCategoryDataRecordFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokTopCategoryFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsGoreFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsLowQualityFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsNsfwFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsOcrFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsSpamFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsViolentFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokPoliticalInclinationFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokSlopScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokSunnyScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokTagsFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableGrokAnnotations +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.home_mixer_features.{thriftscala => hmf} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class GrokAnnotationsFeatureHydrator @Inject() ( + homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GrokAnnotations") + + override val features: Set[Feature[_, _]] = + Set( + GrokAnnotationsFeature, + GrokCategoryDataRecordFeature, + GrokTopCategoryFeature, + GrokTagsFeature, + GrokIsGoreFeature, + GrokIsNsfwFeature, + GrokIsSpamFeature, + GrokIsViolentFeature, + GrokIsLowQualityFeature, + GrokIsOcrFeature, + GrokSunnyScoreFeature, + GrokPoliticalInclinationFeature, + GrokSlopScoreFeature, + DebugStringFeature + ) + + override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableGrokAnnotations) + + private val batchSize = 64 + + private def getGrokAnnotationsFromHMF( + tweetIdsToHydrate: Seq[Long], + ): Future[Seq[Option[hmt.GrokAnnotations]]] = { + val keySerialized = tweetIdsToHydrate.map(_.toString) + val request = hmf.HomeMixerFeaturesRequest( + keySerialized, + hmf.Cache.GrokPostAnnotations + ) + + val responseFut = homeMixerFeatureService.getHomeMixerFeatures(request) + responseFut + .map { response => + response.homeMixerFeatures + .map { homeMixerFeaturesOpt => + homeMixerFeaturesOpt.homeMixerFeaturesType.map { + case hmf.HomeMixerFeaturesType.GrokPostAnnotations(data) => + val metadata = data.annotations.tweetBoolMetadata.map { metadata => + hmt.GrokMetadata( + isNsfw = metadata.isNsfw.getOrElse(false), + isGore = metadata.isGore.getOrElse(false), + isViolent = metadata.isViolent.getOrElse(false), + isSpam = metadata.isSpam.getOrElse(false), + isSoftNsfw = metadata.isSoftNsfw.getOrElse(false), + isLowQuality = metadata.isHighQuality.exists(!_), + isOcr = metadata.isOcr.getOrElse(false), + ) + } + val categoryScoreMap = + data.annotations.entities // This gives Option[List[EntityWithMetadata]] + .map( + _.map(entity => + entity.qualifiedId._2.toString -> entity.score + .getOrElse(0.0) // Convert each entity to (String, Double) + ).toMap + ) // Convert List[(String, Double)] to Map[String, Double] + + hmt.GrokAnnotations( + topics = data.topics, + tags = data.annotations.tags.getOrElse(Seq.empty).map(_.tag), + metadata = metadata, + categoryScores = categoryScoreMap, + sunnyScore = data.annotations.sunnyScore, + politicalInclination = data.annotations.politicalInclination.flatMap { + inclination => + scala.util + .Try(hmt.PoliticalInclination.valueOf(inclination.name)).toOption.flatten + }, + slopScore = data.annotations.slopScore, + ) + case _ => throw new Exception("Unknown type returned") + } + } + }.handle { case _ => Seq.fill(tweetIdsToHydrate.size)(None) } + } + + def getFeatureMaps( + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + ): Future[Seq[FeatureMap]] = { + + val tweetIdsToHydrate = candidates.map(CandidatesUtil.getOriginalTweetId) + val debugStringFeatures = + candidates.map(_.features.getOrElse(DebugStringFeature, None).getOrElse("")) + + val responseMap = getGrokAnnotationsFromHMF(tweetIdsToHydrate) + responseMap.map { result => + result.zip(debugStringFeatures).map { + case (annotations, debugStringFeature) => + val categoryScoreMap: Option[Map[String, Double]] = + annotations.flatMap(_.categoryScores.map(_.toMap)) + val tags: Set[String] = annotations.map(_.tags).getOrElse(Seq.empty).toSet + val grokSlopFeature = annotations.flatMap(_.slopScore.map(_.toLong)) + + FeatureMapBuilder() + .add(GrokAnnotationsFeature, annotations) + .add(GrokCategoryDataRecordFeature, categoryScoreMap) + .add( + GrokTopCategoryFeature, + annotations + .flatMap(_.categoryScores) + .flatMap { scores => + val validCategories = scores.collect { + case (category, score) + if category.forall(_.isDigit) && GrokCategoryIdToNameMap.contains(category.toLong) => + (category.toLong, score) + } + if (validCategories.nonEmpty) { + Some(validCategories.maxBy(_._2)._1) + } else { + None + } + } + ) + .add(GrokTagsFeature, tags.map(_.toLowerCase)) + .add(GrokIsGoreFeature, annotations.flatMap(_.metadata.map(_.isGore))) + .add(GrokIsNsfwFeature, annotations.flatMap(_.metadata.map(_.isNsfw))) + .add(GrokIsSpamFeature, annotations.flatMap(_.metadata.map(_.isSpam))) + .add(GrokIsViolentFeature, annotations.flatMap(_.metadata.map(_.isViolent))) + .add(GrokIsLowQualityFeature, annotations.flatMap(_.metadata.map(_.isLowQuality))) + .add(GrokIsOcrFeature, annotations.flatMap(_.metadata.map(_.isOcr))) + .add(GrokSunnyScoreFeature, annotations.flatMap(_.sunnyScore)) + // Used only for metrics tracking. Does not affect the recommendations. + .add(GrokPoliticalInclinationFeature, annotations.flatMap(_.politicalInclination)) + .add(GrokSlopScoreFeature, grokSlopFeature) + .add( + DebugStringFeature, + Some("%s GrokSlop:%s" + .format(debugStringFeature, grokSlopFeature.map(_.toString).getOrElse("None")))) + .build() + case _ => + FeatureMapBuilder() + .add(GrokAnnotationsFeature, None) + .add(GrokCategoryDataRecordFeature, None) + .add(GrokTopCategoryFeature, None) + .add(GrokTagsFeature, Set.empty[String]) + .add(GrokIsGoreFeature, None) + .add(GrokIsNsfwFeature, None) + .add(GrokIsSpamFeature, None) + .add(GrokIsViolentFeature, None) + .add(GrokIsLowQualityFeature, None) + .add(GrokIsOcrFeature, None) + .add(GrokSunnyScoreFeature, None) + .add(GrokPoliticalInclinationFeature, None) + .add(GrokSlopScoreFeature, None) + .add(DebugStringFeature, None) + .build() + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + OffloadFuturePools.offloadBatchSeqToFutureSeq( + candidates, + getFeatureMaps, + batchSize + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokGorkContentCreatorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokGorkContentCreatorFeatureHydrator.scala new file mode 100644 index 000000000..a4facc634 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokGorkContentCreatorFeatureHydrator.scala @@ -0,0 +1,46 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokContentCreatorFeature +import com.twitter.home_mixer.model.HomeFeatures.GorkContentCreatorFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch + +/** + * Used only for metrics tracking to measure how often we are serving these posts + */ +object GrokGorkContentCreatorFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("GrokGorkContentCreator") + + override val features: Set[Feature[_, _]] = + Set(GrokContentCreatorFeature, GorkContentCreatorFeature) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { + + candidates.map { candidate => + val authorIdOpt = candidate.features.getOrElse(AuthorIdFeature, None) + FeatureMap( + GrokContentCreatorFeature, + authorIdOpt.contains(GrokCreatorId), + GorkContentCreatorFeature, + authorIdOpt.contains(GorkCreatorId), + ) + } + } + + private val GrokCreatorId = // @grok + private val GorkCreatorId = // @gork +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokTranslatedPostIsCachedFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokTranslatedPostIsCachedFeatureHydrator.scala new file mode 100644 index 000000000..cf0cb08a6 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokTranslatedPostIsCachedFeatureHydrator.scala @@ -0,0 +1,76 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.GrokTranslatedPostIsCachedFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableGrokAutoTranslateLanguageFilter +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.searchai.grok.TranslatedPostClientColumn +import com.twitter.search_router.thriftscala.GrokTranslateData +import com.twitter.util.logging.Logging + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class GrokTranslatedPostIsCachedFeatureHydrator @Inject() ( + translatedPostClientColumn: TranslatedPostClientColumn, + statsReceiver: StatsReceiver) + extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] + with Logging { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("GrokTranslatedPostIsCached") + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val successCounter = scopedStatsReceiver.counter("cacheFetch/success") + private val failedCounter = scopedStatsReceiver.counter("cacheFetch/failure") + private val cacheHitCounter = scopedStatsReceiver.counter("cacheHit") + private val cacheMissCounter = scopedStatsReceiver.counter("cacheMiss") + private val DefaultFeatureMap = + FeatureMapBuilder().add(GrokTranslatedPostIsCachedFeature, true).build() + + private val fetcher: Fetcher[(Long, String), Unit, GrokTranslateData] = + translatedPostClientColumn.fetcher + + override val features: Set[Feature[_, _]] = Set(GrokTranslatedPostIsCachedFeature) + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableGrokAutoTranslateLanguageFilter) + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + existingFeatures: FeatureMap + ): Stitch[FeatureMap] = { + val tweetId = candidate.id + val dstLangOpt = query.getLanguageCode + dstLangOpt match { + case Some(dstLang) => + fetcher + .fetch((tweetId, dstLang), ()).map { result => + successCounter.incr() + val cacheAvailable = result.v.isDefined + if (cacheAvailable) cacheHitCounter.incr() + else cacheMissCounter.incr() + FeatureMapBuilder() + .add(GrokTranslatedPostIsCachedFeature, cacheAvailable) + .build() + }.rescue { + case _ => + failedCounter.incr() + Stitch.value(DefaultFeatureMap) + } + case None => Stitch.value(DefaultFeatureMap) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/HeartbeatOptimizerParamsHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/HeartbeatOptimizerParamsHydrator.scala new file mode 100644 index 000000000..d5ed59b2d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/HeartbeatOptimizerParamsHydrator.scala @@ -0,0 +1,84 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.HeartbeatOptimizerParamsMHPkey +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.columns.heartbeat_optimizer.thriftscala.OptimizerMHParamsValue +import com.twitter.strato.columns.heartbeat_optimizer.thriftscala.ParameterAndValue +import com.twitter.strato.generated.client.heartbeat_optimizer.HeartbeatOptimizerParamsMHClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +object HeartbeatOptimizerWeightsFeature extends Feature[PipelineQuery, Option[OptimizerParams]] + +case class OptimizerParams( + epochTimestamp: Long, + nBuckets: Int, + optimizerWeights: Seq[Map[String, Double]]) + +@Singleton +class HeartbeatOptimizerParamsHydrator @Inject() ( + heartbeatOptimizerParamsMHClientColumn: HeartbeatOptimizerParamsMHClientColumn, + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "HeartbeatOptimizerParams") + + private val fetcher: Fetcher[(String, String), Unit, OptimizerMHParamsValue] = + heartbeatOptimizerParamsMHClientColumn.fetcher + + override val features: Set[Feature[_, _]] = Set(HeartbeatOptimizerWeightsFeature) + + private val LKEY = "0" + + private val sucessStat: Counter = statsReceiver.counter("success") + private val failureStat: Counter = statsReceiver.counter("failure") + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val pkey = query.params(HeartbeatOptimizerParamsMHPkey) + fetcher + .fetch((pkey, LKEY)).map { manhattanResult => + manhattanResult.v match { + case Some(optimizerMHParamsValue) => + val parameterAndValueList: Seq[Seq[ParameterAndValue]] = + optimizerMHParamsValue.parameterAndValueList + + val paramMaps: Seq[Map[String, Double]] = parameterAndValueList.map { + paramAndValueList => + paramAndValueList.map { paramAndValue => + (paramAndValue.parameter, paramAndValue.value) + }.toMap + } + val optimizerWeights = OptimizerParams( + epochTimestamp = optimizerMHParamsValue.epochTimestamp.toLong, + nBuckets = paramMaps.size, + optimizerWeights = paramMaps + ) + sucessStat.incr(1) + FeatureMapBuilder() + .add(HeartbeatOptimizerWeightsFeature, Some(optimizerWeights)) + .build() + case _ => + failureStat.incr(1) + FeatureMapBuilder() + .add(HeartbeatOptimizerWeightsFeature, None) + .build() + } + }.handle { + case _ => + failureStat.incr(1) + FeatureMapBuilder() + .add(HeartbeatOptimizerWeightsFeature, None) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/HeavyRankerWeightsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/HeavyRankerWeightsQueryFeatureHydrator.scala new file mode 100644 index 000000000..07ca68da4 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/HeavyRankerWeightsQueryFeatureHydrator.scala @@ -0,0 +1,77 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import breeze.stats.distributions.Beta +import breeze.stats.distributions.Rand +import com.github.nscala_time.time.Imports.LocalDate +import com.twitter.home_mixer.model.PredictedScoreFeature +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.AddNoiseInWeightsPerLabel +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableDailyFrozenNoisyWeights +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NoisyWeightAlphaParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NoisyWeightBetaParam +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import java.lang.{Long => JLong} +import java.util.Objects +import javax.inject.Inject +import javax.inject.Singleton +import com.twitter.ml.api.thriftscala.GeneralTensor +import com.twitter.ml.api.thriftscala.DoubleTensor + +@Singleton +class HeavyRankerWeightsQueryFeatureHydrator @Inject() () + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("HeavyRankerWeights") + + private def doubleToGeneralTensor(value: Double): GeneralTensor = { + val tensor = DoubleTensor(doubles = List(value), shape = Some(List(1L))) + GeneralTensor.DoubleTensor(tensor) + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val builder = FeatureMapBuilder() + // Ensure that for the same userId and start of day, we get the same sequence of random numbers + // Let's us freeze the weight for one day and observe UAS impact + val startOfDay = LocalDate.now().toDateTimeAtStartOfDay.toInstant.getMillis + val seed = + Objects.hash(JLong.valueOf(query.getRequiredUserId), JLong.valueOf(startOfDay)).toLong + val alpha = query.params(NoisyWeightAlphaParam) + val beta = query.params(NoisyWeightBetaParam) + val betaDist = new Beta(alpha, beta) + if (query.params(EnableDailyFrozenNoisyWeights)) Rand.generator.setSeed(seed) + for (predictedScoreFeature <- PredictedScoreFeature.PredictedScoreFeatures) { + val presetWeight = query.params(predictedScoreFeature.modelWeightParam) + val weight = if (query.params(AddNoiseInWeightsPerLabel)) { + // Apply noise to each weight + presetWeight * (1 + betaDist.draw()) + } else { + presetWeight + } + builder.add(predictedScoreFeature.weightQueryFeature, Some(weight)) + for (biasQueryFeature <- predictedScoreFeature.biasQueryFeature; + modelBiasParam <- predictedScoreFeature.modelBiasParam) { + builder.add(biasQueryFeature, Some(query.params(modelBiasParam))) + } + for (debiasQueryFeature <- predictedScoreFeature.debiasQueryFeature; + modelDebiasParam <- predictedScoreFeature.modelDebiasParam) { + val debiasValue = query.params(modelDebiasParam) + builder.add(debiasQueryFeature, Some(doubleToGeneralTensor((debiasValue)))) + } + } + Stitch.value(builder.build()) + } + + override val features: Set[Feature[_, _]] = { + val weightFeatures = PredictedScoreFeature.PredictedScoreFeatureSet.map(_.weightQueryFeature) + val biasFeatures = PredictedScoreFeature.PredictedScoreFeatureSet.flatMap(_.biasQueryFeature) + val debiasFeatures = PredictedScoreFeature.PredictedScoreFeatureSet + .flatMap(_.debiasQueryFeature) + (weightFeatures ++ biasFeatures ++ debiasFeatures).toSet + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressedImageClusterIdsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressedImageClusterIdsQueryFeatureHydrator.scala new file mode 100644 index 000000000..3ee0f3d72 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressedImageClusterIdsQueryFeatureHydrator.scala @@ -0,0 +1,112 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.ImageClipClusterIdInMemCache +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.impressionstore.thriftscala.ImpressionList +import com.twitter.strato.client.Client +import com.twitter.strato.client.Fetcher +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +case object ImpressedImageClusterIds extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { + override val defaultValue: Seq[Long] = Seq.empty +} + +/** + * Get the list of image cluster ids that the user has already seen. + */ +@Singleton +case class ImpressedImageClusterIdsQueryFeatureHydrator @Inject() ( + stratoClient: Client, + tweetImpressionStore: ReadableStore[Long, ImpressionList], + @Named(ImageClipClusterIdInMemCache) imageClipClusterIdInMemCache: InProcessCache[ + Long, + Option[Option[Long]] + ], + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "ImpressedImageClusterIdsQuery") + + override val features: Set[Feature[_, _]] = Set(ImpressedImageClusterIds) + + val clusterIdFetcher: Fetcher[Long, Unit, Long] = + stratoClient.fetcher[Long, Unit, Long]( + "videoRecommendations/twitterClip/twitterClipImageClusterIdMh95") + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("impressionKey/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("impressionKey/notFound") + private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") + private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") + private val fetchExceptionCounter = + scopedStatsReceiver.counter("getFromCacheOrFetch/exception") + + private def getImageClipClusterId(tweetId: Long): Stitch[Option[Option[Long]]] = { + imageClipClusterIdInMemCache + .get(tweetId) + .map { cachedValue => + cacheHitCounter.incr() + Stitch.value(cachedValue) + }.getOrElse { + cacheMissCounter.incr() + clusterIdFetcher + .fetch(tweetId) + .map(_.v) + .liftToOption() + .flatMap { clusterIdOpt => + imageClipClusterIdInMemCache.set(tweetId, clusterIdOpt) + Stitch.value(clusterIdOpt) + } + }.handle { + case _: Exception => + fetchExceptionCounter.incr() + None + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + query.getOptionalUserId match { + case Some(userId) => + val featureMapResult: Future[Seq[Long]] = tweetImpressionStore + .get(userId).map { impressionListOpt => + if (impressionListOpt.isEmpty) { + keyNotFoundCounter.incr() + } + keyFoundCounter.incr() + + val tweetIdsOpt = for { + impressionList <- impressionListOpt + impressions <- impressionList.impressions + } yield { + impressions.take(250).map(_.tweetId) + } + tweetIdsOpt.getOrElse(Seq.empty) + } + + Stitch.callFuture(featureMapResult).flatMap { tweetIds => + Stitch + .traverse(tweetIds) { tweetId => + getImageClipClusterId(tweetId) + }.map { results: Seq[Option[Option[Long]]] => + FeatureMapBuilder().add(ImpressedImageClusterIds, results.flatten.flatten).build() + } + } + + case None => + val featureMapResult = FeatureMapBuilder().add(ImpressedImageClusterIds, Seq.empty).build() + Stitch.value(featureMapResult) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressedMediaClusterIdsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressedMediaClusterIdsQueryFeatureHydrator.scala new file mode 100644 index 000000000..0afacbf45 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressedMediaClusterIdsQueryFeatureHydrator.scala @@ -0,0 +1,113 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClusterId95Store +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClipClusterIdInMemCache +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.impressionstore.thriftscala.ImpressionList +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +case object ImpressedMediaClusterIds extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { + override val defaultValue: Seq[Long] = Seq.empty +} + +/** + * Get the list of media cluster ids that the user has already seen. + */ +@Singleton +case class ImpressedMediaClusterIdsQueryFeatureHydrator @Inject() ( + @Named(MediaClusterId95Store) clusterIdStore: ReadableStore[Long, Long], + tweetImpressionStore: ReadableStore[Long, ImpressionList], + @Named(MediaClipClusterIdInMemCache) mediaClipClusterIdInMemCache: InProcessCache[ + Long, + Option[Option[Long]] + ], + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "ImpressedMediaClusterIdsQuery") + + override val features: Set[Feature[_, _]] = Set(ImpressedMediaClusterIds) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("impressionKey/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("impressionKey/notFound") + private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") + private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") + private val storeHitCounter = scopedStatsReceiver.counter("store/hit") + private val storeMissCounter = scopedStatsReceiver.counter("store/miss") + private val fetchExceptionCounter = scopedStatsReceiver.counter("fetch/exception") + + private def getmediaClipClusterId(tweetId: Long): Stitch[Option[Option[Long]]] = { + mediaClipClusterIdInMemCache + .get(tweetId) + .map { cachedValue => + cacheHitCounter.incr() + Stitch.value(cachedValue) + }.getOrElse { + cacheMissCounter.incr() + Stitch + .callFuture(clusterIdStore.get(tweetId)) + .flatMap { clusterIdOpt => + if (clusterIdOpt.isDefined) { + storeHitCounter.incr() + } else { + storeMissCounter.incr() + } + val wrappedClusterIdOpt = Some(clusterIdOpt) + mediaClipClusterIdInMemCache.set(tweetId, wrappedClusterIdOpt) + Stitch.value(wrappedClusterIdOpt) + } + }.handle { + case _: Exception => + fetchExceptionCounter.incr() + None + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + query.getOptionalUserId match { + case Some(userId) => + val featureMapResult: Future[Seq[Long]] = tweetImpressionStore + .get(userId).map { impressionListOpt => + if (impressionListOpt.isEmpty) { + keyNotFoundCounter.incr() + } + keyFoundCounter.incr() + + val tweetIdsOpt = for { + impressionList <- impressionListOpt + impressions <- impressionList.impressions + } yield { + impressions.take(250).map(_.tweetId) + } + tweetIdsOpt.getOrElse(Seq.empty) + } + + Stitch.callFuture(featureMapResult).flatMap { tweetIds => + Stitch + .traverse(tweetIds) { tweetId => + getmediaClipClusterId(tweetId) + }.map { results: Seq[Option[Option[Long]]] => + FeatureMapBuilder().add(ImpressedMediaClusterIds, results.flatten.flatten).build() + } + } + + case None => + val featureMapResult = FeatureMapBuilder().add(ImpressedMediaClusterIds, Seq.empty).build() + Stitch.value(featureMapResult) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala index 6ece16ce0..342d68ddd 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala @@ -1,9 +1,7 @@ package com.twitter.home_mixer.functional_component.feature_hydrator -import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MemcachedImpressionBloomFilterStore import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -12,51 +10,37 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch -import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient +import com.twitter.storehaus.Store import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} -import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter import javax.inject.Inject +import javax.inject.Named import javax.inject.Singleton @Singleton -case class ImpressionBloomFilterQueryFeatureHydrator[ - Query <: PipelineQuery with HasSeenTweetIds] @Inject() ( - bloomFilterClient: ManhattanStoreClient[ +case class ImpressionBloomFilterQueryFeatureHydrator @Inject() ( + @Named(MemcachedImpressionBloomFilterStore) bloomFilterClient: Store[ blm.ImpressionBloomFilterKey, blm.ImpressionBloomFilterSeq - ]) extends QueryFeatureHydrator[Query] { + ]) extends QueryFeatureHydrator[PipelineQuery] { - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( - "ImpressionBloomFilter") - - private val ImpressionBloomFilterTTL = 7.day + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ImpressionBloomFilter") override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature) private val SurfaceArea = blm.SurfaceArea.HomeTimeline - override def hydrate(query: Query): Stitch[FeatureMap] = { + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId - bloomFilterClient - .get(blm.ImpressionBloomFilterKey(userId, SurfaceArea)) - .map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty))) - .map { bloomFilterSeq => - val updatedBloomFilterSeq = - if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq - else { - ImpressionBloomFilter.addSeenTweetIds( - surfaceArea = SurfaceArea, - tweetIds = query.seenTweetIds.get, - bloomFilterSeq = bloomFilterSeq, - timeToLive = ImpressionBloomFilterTTL, - falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam) - ) - } - FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build() - } + Stitch.callFuture { + bloomFilterClient + .get(blm.ImpressionBloomFilterKey(userId, SurfaceArea)) + .map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty))) + .map { bloomFilterSeq => + FeatureMapBuilder().add(ImpressionBloomFilterFeature, bloomFilterSeq).build() + } + } } - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) + override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99)) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNegativeFeedbackTimeQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNegativeFeedbackTimeQueryFeatureHydrator.scala new file mode 100644 index 000000000..d3cdf739a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNegativeFeedbackTimeQueryFeatureHydrator.scala @@ -0,0 +1,75 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.LastNegativeFeedbackTimeFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.recommendations.user_signal_service.SignalsClientColumn +import com.twitter.usersignalservice.thriftscala.BatchSignalRequest +import com.twitter.usersignalservice.thriftscala.ClientIdentifier +import com.twitter.usersignalservice.thriftscala.SignalRequest +import com.twitter.usersignalservice.thriftscala.SignalType +import com.twitter.usersignalservice.thriftscala.{Signal => UssSignal} +import com.twitter.util.Time +import com.twitter.util.logging.Logging +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class LastNegativeFeedbackTimeQueryFeatureHydrator @Inject() ( + signalsClientColumn: SignalsClientColumn) + extends QueryFeatureHydrator[PipelineQuery] + with Logging { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("USSSignals") + private val fetcher = signalsClientColumn.fetcher + + override val features: Set[Feature[_, _]] = Set( + LastNegativeFeedbackTimeFeature, + ) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val lastNegativeFeedbackTime = getLastNegativeFeedbackTime(query.getRequiredUserId) + lastNegativeFeedbackTime.map { FeatureMap(LastNegativeFeedbackTimeFeature, _) } + } + + def getLastNegativeFeedbackTime(userId: Long): Stitch[Option[Time]] = { + val enabledNegativeSignalTypes = Seq( + SignalType.AccountBlock, + SignalType.AccountMute, + SignalType.TweetSeeFewer, + SignalType.TweetReport, + SignalType.TweetDontLike) + + // negative signals + val maybeNegativeSignals = + enabledNegativeSignalTypes.map { negativeSignal => + SignalRequest( + maxResults = Some(1), // Only most recent needed + signalType = negativeSignal + ) + } + + val batchSignalRequest = + BatchSignalRequest(userId, maybeNegativeSignals, Some(ClientIdentifier.CrMixerHome)) + + val signalsStitch = fetcher + .fetch(batchSignalRequest) + .map { result => + result.v + .map(_.signalResponse.toSeq.flatMap { + case (_, signals) => + signals + }).getOrElse(Seq.empty) + } + val getTimestamp: UssSignal => Option[Time] = signal => + if (signal.timestamp == 0L) None else Some(Time.fromMilliseconds(signal.timestamp)) + signalsStitch.map { + _.map(getTimestamp).flatten.reduceOption((a, b) => if (a < b) a else b) + } + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ListIdsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ListIdsQueryFeatureHydrator.scala new file mode 100644 index 000000000..b0bcaa9a4 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ListIdsQueryFeatureHydrator.scala @@ -0,0 +1,61 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.param.HomeGlobalParams.ListMandarinTweetsParams.ListMandarinTweetsEnable +import com.twitter.home_mixer.param.HomeGlobalParams.ListMandarinTweetsParams.ListMandarinTweetsLists +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.socialgraph.{thriftscala => sg} +import com.twitter.stitch.Stitch +import com.twitter.stitch.socialgraph.SocialGraph + +import javax.inject.Inject +import javax.inject.Singleton + +case object ListIdsFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { + override val defaultValue: Seq[Long] = Seq.empty +} + +@Singleton +class ListIdsQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ListIds") + + override val features: Set[Feature[_, _]] = Set(ListIdsFeature) + + private val MaxListsToFetch = 20 + + private def buildIdsRequest(userId: Long, relationshipType: sg.RelationshipType) = sg.IdsRequest( + relationships = Seq( + sg.SrcRelationship(userId, relationshipType, hasRelationship = true), + sg.SrcRelationship(userId, sg.RelationshipType.ListMuting, hasRelationship = false) + ), + pageRequest = Some(sg.PageRequest(selectAll = Some(false), count = Some(MaxListsToFetch))), + context = Some(sg.LookupContext(performUnion = Some(false), includeAll = Some(false))) + ) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + + val subscribedRequest = buildIdsRequest(userId, sg.RelationshipType.ListIsSubscriber) + val ownedRequest = buildIdsRequest(userId, sg.RelationshipType.ListOwning) + + Stitch.join(socialGraph.ids(ownedRequest), socialGraph.ids(subscribedRequest)).map { + case (ownedResponse, subscribedResponse) => + val recommendedListIds = + if (query.params(ListMandarinTweetsEnable)) + query.params(ListMandarinTweetsLists) + else + Seq.empty + + val ids = (ownedResponse.ids ++ subscribedResponse.ids ++ recommendedListIds).distinct + + FeatureMapBuilder().add(ListIdsFeature, ids).build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MediaClusterIdFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MediaClusterIdFeatureHydrator.scala new file mode 100644 index 000000000..69a34afa8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MediaClusterIdFeatureHydrator.scala @@ -0,0 +1,123 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.MediaCategoryFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaClusterIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClipClusterIdInMemCache +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClusterId95Store +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMediaClusterFeatureHydrationParam +import com.twitter.mediaservices.commons.thriftscala.MediaCategory +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch + +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import com.twitter.storehaus.ReadableStore + +@Singleton +class MediaClusterIdFeatureHydrator @Inject() ( + @Named(MediaClusterId95Store) clusterIdStore: ReadableStore[Long, Long], + @Named(MediaClipClusterIdInMemCache) mediaClipClusterIdInMemCache: InProcessCache[ + Long, + Option[Option[Long]] + ], + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] + with WithDefaultFeatureMap { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("MediaClusterId") + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableMediaClusterFeatureHydrationParam) + + override val features: Set[Feature[_, _]] = Set(TweetMediaClusterIdsFeature) + + override val defaultFeatureMap: FeatureMap = + FeatureMap(TweetMediaClusterIdsFeature, Map.empty[Long, Long]) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") + private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") + private val fetchExceptionCounter = + scopedStatsReceiver.counter("getFromCacheOrFetch/exception") + + private def getFromCacheOrFetch(tweetId: Long): Stitch[Option[Option[Long]]] = { + mediaClipClusterIdInMemCache + .get(tweetId) + .map { cachedValue => + cacheHitCounter.incr() + Stitch.value(cachedValue) + }.getOrElse { + cacheMissCounter.incr() + // Use the store which handles memcache + Manhattan fallback + Stitch + .callFuture(clusterIdStore.get(tweetId)).map { result => + result match { + case Some(clusterId) => + keyFoundCounter.incr() + val finalResult = Some(Some(clusterId)) + mediaClipClusterIdInMemCache.set(tweetId, finalResult) + finalResult + case None => + keyNotFoundCounter.incr() + mediaClipClusterIdInMemCache.set(tweetId, Some(None)) + Some(None) + } + }.handle { + case ex: Exception => + fetchExceptionCounter.incr() + ex.printStackTrace() + None + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + Stitch.collect( + candidates.map { candidate => + val mediaIds = candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty[Long]) + val mediaCategory = candidate.features.getOrElse(MediaCategoryFeature, None) + if (mediaCategory.contains(MediaCategory.TweetVideo) || mediaCategory.contains( + MediaCategory.AmplifyVideo)) { + val items: Seq[Stitch[Option[(Long, Long)]]] = mediaIds.map { mediaId => + getFromCacheOrFetch(candidate.candidate.id).map { + case Some(Some(mediaClusterId)) => + keyFoundCounter.incr() + Some(mediaId -> mediaClusterId) + case _ => + keyNotFoundCounter.incr() + None + } + } + + Stitch.collect(items).map { results => + val mediaClusterMap = + results.flatten.toMap // Flatten to remove None and create Map[Long, Long] + FeatureMapBuilder().add(TweetMediaClusterIdsFeature, mediaClusterMap).build() + } + } else { + Stitch.value( + FeatureMapBuilder().add(TweetMediaClusterIdsFeature, Map.empty[Long, Long]).build()) + } + } + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MediaCompletionRateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MediaCompletionRateFeatureHydrator.scala new file mode 100644 index 000000000..0784d758d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MediaCompletionRateFeatureHydrator.scala @@ -0,0 +1,106 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaCompletionRateFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaCompletionRateInMemCache +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Client +import com.twitter.strato.generated.client.analytics.video.VideoCompletionRateOnApiMediaClientColumn +import com.twitter.strato.graphql.thriftscala.ApiMediaKey.TweetVideo +import com.twitter.strato.graphql.thriftscala.TweetVideoKey +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class MediaCompletionRateFeatureHydrator @Inject() ( + @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client, + @Named(MediaCompletionRateInMemCache) mediaCompletionRateInMemCache: InProcessCache[Long, Double], + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("MediaCompletionRate") + + override val features: Set[Feature[_, _]] = Set(TweetMediaCompletionRateFeature) + + private val completionRateFetcher = stratoClient.fetcher[ + VideoCompletionRateOnApiMediaClientColumn.Key, + VideoCompletionRateOnApiMediaClientColumn.View, + VideoCompletionRateOnApiMediaClientColumn.Value + ](VideoCompletionRateOnApiMediaClientColumn.Path) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") + private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") + private val fetchExceptionCounter = + scopedStatsReceiver.counter("getFromCacheOrFetch/exception") + + private def getFromCacheOrFetch(mediaId: Long): Stitch[Option[Double]] = { + mediaCompletionRateInMemCache + .get(mediaId) + .map { cachedValue => + cacheHitCounter.incr() + Stitch.value(Some(cachedValue)) + }.getOrElse { + cacheMissCounter.incr() + val key = TweetVideo(TweetVideoKey(mediaId = mediaId, tweetId = 0L)) + completionRateFetcher + .fetch(key) + .flatMap { result => + val completionRate = result.v.map(_.organic) + mediaCompletionRateInMemCache.set(mediaId, completionRate.getOrElse(0.0)) + Stitch.value(completionRate) + } + }.handle { + case _: Exception => + fetchExceptionCounter.incr() + None + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val featureMaps = candidates.map { candidate => + val hasVideo = candidate.features.getOrElse(HasVideoFeature, false) + val mediaIds = candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty[Long]) + val inNetwork = candidate.features.getOrElse(FromInNetworkSourceFeature, false) + val completionRates = if (hasVideo && !inNetwork) { + mediaIds.map { mediaId => + getFromCacheOrFetch(mediaId).map { + case Some(completionRate) => + keyFoundCounter.incr() + Some(completionRate) + case _ => + keyNotFoundCounter.incr() + None + } + } + } else Seq.empty[Stitch[Option[Double]]] + + Stitch.collectToTry(completionRates).map { results => + val completionRates = results.map(_.toOption.flatten).flatten + val maxCompletionRate = if (completionRates.nonEmpty) Some(completionRates.max) else None + FeatureMap(TweetMediaCompletionRateFeature, maxCompletionRate) + } + } + Stitch.collect(featureMaps) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MultiModalEmbeddingsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MultiModalEmbeddingsFeatureHydrator.scala new file mode 100644 index 000000000..d0cf83728 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MultiModalEmbeddingsFeatureHydrator.scala @@ -0,0 +1,107 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.MultiModalEmbeddingsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ExpiringLruInProcessCache +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.searchai.storage.PostMultimodalEmbeddingMhClientColumn +import com.twitter.util.Duration +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Random +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.util.Try + +object MultiModalEmbeddingsFeatureHydrator { + private val BaseTTL = 15 + private val TTL: Duration = (BaseTTL + Random.nextInt(10)).minutes + private val maximumCacheSize = 15000 // 15000 * 1536 * 8 bytes = <200Mb + + val cache: InProcessCache[Long, Option[Option[Seq[Double]]]] = + new ExpiringLruInProcessCache(ttl = TTL, maximumSize = maximumCacheSize) + + private val embeddingFetcherModelVersion = "v1" +} + +@Singleton +class MultiModalEmbeddingsFeatureHydrator @Inject() ( + postMultimodalEmbeddingMhClientColumn: PostMultimodalEmbeddingMhClientColumn, + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("MultiModalEmbeddings") + + override val features: Set[Feature[_, _]] = Set(MultiModalEmbeddingsFeature) + + private val embeddingFetcher = postMultimodalEmbeddingMhClientColumn.fetcher + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + private val fetchExceptionCounter = scopedStatsReceiver.counter("getFromCacheOrFetch/exception") + + private def getEmbedding( + tweetId: Long + ): Stitch[Option[Option[Seq[Double]]]] = { + val embeddingStitchFromCache: Option[Stitch[Option[Option[Seq[Double]]]]] = + MultiModalEmbeddingsFeatureHydrator.cache + .get(tweetId) + .map(Stitch.value) + + embeddingStitchFromCache match { + case Some(stitch: Stitch[Option[Option[Seq[Double]]]]) => stitch + case None => + val embeddingStitch = embeddingFetcher + .fetch((tweetId, MultiModalEmbeddingsFeatureHydrator.embeddingFetcherModelVersion)).map { + result => + result.v match { + case Some(tweetEmbedding) + if tweetEmbedding != null && tweetEmbedding.embedding1 != null && tweetEmbedding.embedding1.isDefined => + keyFoundCounter.incr() + val embedding = tweetEmbedding.embedding1 + MultiModalEmbeddingsFeatureHydrator.cache.set(tweetId, Some(embedding)) + Some(embedding) + case _ => + keyNotFoundCounter.incr() + MultiModalEmbeddingsFeatureHydrator.cache.set(tweetId, Some(None)) + None + } + }.handle { + case ex: Exception => + fetchExceptionCounter.incr() + None + } + embeddingStitch + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val stitchesWithTry: Stitch[Seq[Try[Option[Option[Seq[Double]]]]]] = Stitch.collectToTry { + candidates.map { candidate => + val tweetId: Long = candidate.candidate.id + getEmbedding(tweetId) + } + } + + stitchesWithTry.map { tryVal => + tryVal.flatMap(_.toOption).map { + case Some(embedding) => FeatureMap(MultiModalEmbeddingsFeature, embedding) + case _ => FeatureMap(MultiModalEmbeddingsFeature, None) + } + } + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.scala index ede075e5c..d8a746bbd 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.scala @@ -8,7 +8,6 @@ import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.param.HomeGlobalParams.EnableNahFeedbackInfoParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -36,7 +35,7 @@ class NamesFeatureHydrator @Inject() (gizmoduck: Gizmoduck) override val features: Set[Feature[_, _]] = Set(ScreenNamesFeature, RealNamesFeature) override def onlyIf(query: PipelineQuery): Boolean = query.product match { - case FollowingProduct => query.params(EnableNahFeedbackInfoParam) + case FollowingProduct => false case _ => true } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NaviClientConfigQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NaviClientConfigQueryFeatureHydrator.scala new file mode 100644 index 000000000..0cead6d25 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NaviClientConfigQueryFeatureHydrator.scala @@ -0,0 +1,72 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.NaviClientConfigFeature +import com.twitter.home_mixer.model.NaviClientConfig +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelIdParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NaviGPUBatchSizeParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ProdModelIdParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseGPUNaviClusterParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseGPUNaviClusterTestUsersParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseRealtimeNaviClusterParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseSecondaryNaviClusterParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecap +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapGPU +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapRealtime +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapSecondary +import com.twitter.product_mixer.component_library.module.TestUserMapper +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class NaviClientConfigQueryFeatureHydrator @Inject() ( + testUserMapper: TestUserMapper) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("NaviClientConfig") + + override val features: Set[Feature[_, _]] = Set(NaviClientConfigFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + // Use experimental realtime cluster + val useRealtimeCluster = query.params(UseRealtimeNaviClusterParam) + // Use secondary navi cluster for handling peak traffic + val useSecondaryCluster = query.params(UseSecondaryNaviClusterParam) + + val modelId = query.params(ModelIdParam) + val prodModelId = query.params(ProdModelIdParam) + val isTestUser = testUserMapper.isTestUser(query.clientContext) + val useGPUCluster = prodModelId == modelId && (query.params(UseGPUNaviClusterParam) || + (isTestUser && query.params(UseGPUNaviClusterTestUsersParam))) + + val gpuBatchSize = + if (useGPUCluster) Some(query.params(NaviGPUBatchSizeParam).toInt) + else None + + val clusterClientPriorityList = Seq( + (useGPUCluster, NaviModelClientHomeRecapGPU), + (useRealtimeCluster, NaviModelClientHomeRecapRealtime), + (useSecondaryCluster, NaviModelClientHomeRecapSecondary) + ) + + val clientName = clusterClientPriorityList + .collectFirst { + case (true, name) => name + }.getOrElse(NaviModelClientHomeRecap) + + val clusterStr = clientName match { + case NaviModelClientHomeRecapGPU => "GPU" + case NaviModelClientHomeRecapRealtime => "Realtime" + case _ => "" + } + + val naviConfig = NaviClientConfig(clientName, gpuBatchSize, clusterStr) + Stitch.value(FeatureMap(NaviClientConfigFeature, naviConfig)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NaviVideoClientConfigQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NaviVideoClientConfigQueryFeatureHydrator.scala new file mode 100644 index 000000000..39c80041a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NaviVideoClientConfigQueryFeatureHydrator.scala @@ -0,0 +1,38 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.NaviClientConfigFeature +import com.twitter.home_mixer.model.NaviClientConfig +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring._ +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapVideo +import com.twitter.product_mixer.component_library.module.TestUserMapper +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class NaviVideoClientConfigQueryFeatureHydrator @Inject() ( + testUserMapper: TestUserMapper) + extends NaviClientConfigQueryFeatureHydrator(testUserMapper) { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("NaviVideoClientConfig") + + override val features: Set[Feature[_, _]] = Set(NaviClientConfigFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + query.params(UseVideoNaviClusterParam) match { + case true => + val config = NaviClientConfig( + clientName = NaviModelClientHomeRecapVideo, + customizedBatchSize = None, + clusterStr = "Video" + ) + Stitch.value(FeatureMap(NaviClientConfigFeature, config)) + case _ => super.hydrate(query) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OnPremRealGraphQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OnPremRealGraphQueryFeatureHydrator.scala new file mode 100644 index 000000000..7be4fb095 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OnPremRealGraphQueryFeatureHydrator.scala @@ -0,0 +1,49 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableOnPremRealGraphQueryFeatures +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.recommendations.interaction_graph.on_prem.RealGraphUserFeaturesOnUserClientColumn +import com.twitter.timelines.real_graph.{thriftscala => rg} + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class OnPremRealGraphQueryFeatureHydrator @Inject() ( + realGraphUserFeaturesClientColumn: RealGraphUserFeaturesOnUserClientColumn) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("OnPremRealGraphFeatures") + + override val features: Set[Feature[_, _]] = Set(RealGraphFeatures) + + override def onlyIf( + query: PipelineQuery + ): Boolean = { + query.params(EnableOnPremRealGraphQueryFeatures) + } + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + realGraphUserFeaturesClientColumn.fetcher + .fetch(query.getRequiredUserId).map { userSession => + val realGraphFeaturesMap = userSession.v.flatMap { userSession => + userSession.realGraphFeatures.collect { + case rg.RealGraphFeatures.V1(realGraphFeatures) => + val edgeFeatures = + realGraphFeatures.edgeFeatures ++ realGraphFeatures.oonEdgeFeatures + edgeFeatures.map { edge => edge.destId -> edge }.toMap + } + } + + FeatureMapBuilder().add(RealGraphFeatures, realGraphFeaturesMap).build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OptimizerWeightsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OptimizerWeightsQueryFeatureHydrator.scala new file mode 100644 index 000000000..6bf0ff344 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OptimizerWeightsQueryFeatureHydrator.scala @@ -0,0 +1,93 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.PredictedScoreFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import javax.inject.Inject + +/* + * This Hydrator updates the model scoring weights with the optimizer weights. + */ + +class OptimizerWeightsQueryFeatureHydrator @Inject() ( + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("OptimizerWeights") + + private val EnabledPredictedScoreFeatures: Seq[PredictedScoreFeature] = + PredictedScoreFeature.PredictedScoreFeatures + + override val features: Set[Feature[_, _]] = { + EnabledPredictedScoreFeatures.map(_.weightQueryFeature).toSet + } + + private val BIG_PRIME = 65537 + private def hashFn(userId: Long, seed: Long, numBuckets: Int): Int = { + val x = userId % BIG_PRIME + val y = seed % BIG_PRIME + val h = (31 * x + 41 * y) * 53 + (h % BIG_PRIME).toInt % numBuckets + } + + private val successCounter = + statsReceiver.counter("success") + private val failureCounter = statsReceiver.counter("failure") + + private def get_user_weights( + userId: Long, + optimizerParams: OptimizerParams + ): Map[String, Double] = { + val weightIndex = + hashFn(userId, seed = optimizerParams.epochTimestamp, numBuckets = optimizerParams.nBuckets) + val userWeights = optimizerParams.optimizerWeights(weightIndex) + userWeights + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val optimizerParams = query.features + .flatMap(_.getOrElse(HeartbeatOptimizerWeightsFeature, None)) + + val featureMapBuilder = FeatureMapBuilder() + + optimizerParams match { + case Some(optimizerParams) => + if (optimizerParams.optimizerWeights.size != optimizerParams.nBuckets) { + failureCounter.incr(1) + // Fall back to default weights when size mismatch + for (predictedScoreFeature <- EnabledPredictedScoreFeatures) { + val currentWeight = query.params(predictedScoreFeature.modelWeightParam) + featureMapBuilder.add(predictedScoreFeature.weightQueryFeature, Some(currentWeight)) + } + } else { + val userWeights: Map[String, Double] = + get_user_weights(query.getRequiredUserId, optimizerParams) + + for (predictedScoreFeature <- EnabledPredictedScoreFeatures) { + val currentWeight = query.params(predictedScoreFeature.modelWeightParam) + val weight_name = predictedScoreFeature.modelWeightParam.name + val optimizerWeight = + userWeights.getOrElse(weight_name, currentWeight) + featureMapBuilder.add(predictedScoreFeature.weightQueryFeature, Some(optimizerWeight)) + } + successCounter.incr(1) + } + + case _ => + // When HeartbeatOptimizerWeightsFeature is None, use default weights + for (predictedScoreFeature <- EnabledPredictedScoreFeatures) { + val currentWeight = query.params(predictedScoreFeature.modelWeightParam) + featureMapBuilder.add(predictedScoreFeature.weightQueryFeature, Some(currentWeight)) + } + failureCounter.incr(1) + } + + Stitch.value(featureMapBuilder.build()) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OriginalAuthorLargeEmbeddingsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OriginalAuthorLargeEmbeddingsFeatureHydrator.scala new file mode 100644 index 000000000..66b1af6fa --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OriginalAuthorLargeEmbeddingsFeatureHydrator.scala @@ -0,0 +1,161 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.OriginalAuthorLargeEmbeddingsFeature +import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.OriginalAuthorLargeEmbeddingsKeyFeature +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam +import com.twitter.home_mixer.util.CandidatesUtil.getOriginalAuthorId +import com.twitter.home_mixer_features.{thriftscala => hmf} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.adapters.large_embeddings.HashingFeatureParams +import com.twitter.timelines.prediction.adapters.large_embeddings.HomeMixerLargeEmbeddingsFeatureHydrator +import com.twitter.timelines.prediction.adapters.large_embeddings.LargeEmbeddingsAdapter +import com.twitter.timelines.prediction.adapters.large_embeddings.OriginalAuthorLargeEmbeddingsAdapter +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class OriginalAuthorLargeEmbeddingsFeatureHydrator @Inject() ( + statsReceiver: StatsReceiver, + override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] + with HomeMixerLargeEmbeddingsFeatureHydrator { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("OriginalAuthorLargeEmbeddings") + + override val features: Set[Feature[_, _]] = + Set(OriginalAuthorLargeEmbeddingsFeature, OriginalAuthorLargeEmbeddingsKeyFeature) + + override val adapter: LargeEmbeddingsAdapter = OriginalAuthorLargeEmbeddingsAdapter + + override val cacheType: hmf.Cache = hmf.Cache.AuthorLargeEmbeddings + + override val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableLargeEmbeddingsFeatureHydrationParam) + + // Hashing Features + override val defaultHashingFeatureParams: HashingFeatureParams = HashingFeatureParams( + scales = Seq(3384241453L, 3372414709L), + biases = Seq(1649585795L, 3131243219L), + modulus = 3957384397L, + bucketSize = 3000000L, + ) + + override val modelName2HashingFeatureParams: Map[String, HashingFeatureParams] = Map( + "hr_video_prod__v3_realtime" -> HashingFeatureParams( + scales = Seq(113294449L, 601841083L), + biases = Seq(2231299001L, 841367196L), + modulus = 2343760591L, + bucketSize = 3000000L, + ), + "hr_video_prod__v2_lembeds" -> HashingFeatureParams( + scales = Seq(787140070L, 633713480L), + biases = Seq(427768658L, 911091889L), + modulus = 2888480981L, + bucketSize = 300000L, + ), + "hr_prod__v4_embeds_230M" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + "hr_prod__v5_embeds_230M_and_transformer" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + "hr_prod__v5_watchtime" -> HashingFeatureParams( + scales = Seq(2328530078L, 2844016377L), + biases = Seq(1352496802L, 3011003330L), + modulus = 3979826519L, + bucketSize = 30000000L, + ), + "hr_prod__v6_transformer_v2" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + "hr_prod__v6_mixed_training" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + "hr_prod__v6_transformer_v2_kafka_merge_join" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + "hr_prod__v6_transformer_v2_realtime_debias_21apr" -> HashingFeatureParams( + scales = Seq(371965780L, 328930218L), + biases = Seq(139686260L, 37755056L), + modulus = 631860353L, + bucketSize = 30000000L, + ), + "hr_video_prod__v4_realtime" -> HashingFeatureParams( + scales = Seq(113294449L, 601841083L), + biases = Seq(2231299001L, 841367196L), + modulus = 2343760591L, + bucketSize = 3000000L, + ), + "hr_video_prod__v4_realtime_mergehead" -> HashingFeatureParams( + scales = Seq(113294449L, 601841083L), + biases = Seq(2231299001L, 841367196L), + modulus = 2343760591L, + bucketSize = 3000000L, + ), + ) + + private val batchSize = 25 + + private def getBatchedFeatureMap( + modelName: String, + candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], + ): Future[Seq[FeatureMap]] = { + val originalAuthorIds = + candidatesBatch.map { candidate => + getOriginalAuthorId(candidate.features).getOrElse(0L) + } + + getLargeEmbeddings(originalAuthorIds, modelName).map { responses => + responses.map { largeEmbeddingResponse => + FeatureMapBuilder() + .add(OriginalAuthorLargeEmbeddingsFeature, largeEmbeddingResponse.dataRecord) + .add(OriginalAuthorLargeEmbeddingsKeyFeature, largeEmbeddingResponse.hashedKeys) + .build() + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val modelName = query.params(ModelNameParam) + OffloadFuturePools.offloadBatchSeqToFutureSeq( + candidates, + getBatchedFeatureMap(modelName, _), + batchSize + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OriginalTweetLargeEmbeddingsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OriginalTweetLargeEmbeddingsFeatureHydrator.scala new file mode 100644 index 000000000..d5b3697e5 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OriginalTweetLargeEmbeddingsFeatureHydrator.scala @@ -0,0 +1,161 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.OriginalTweetLargeEmbeddingsFeature +import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.OriginalTweetLargeEmbeddingsKeyFeature +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam +import com.twitter.home_mixer.util.CandidatesUtil.getOriginalTweetId +import com.twitter.home_mixer_features.{thriftscala => hmf} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.adapters.large_embeddings.HashingFeatureParams +import com.twitter.timelines.prediction.adapters.large_embeddings.HomeMixerLargeEmbeddingsFeatureHydrator +import com.twitter.timelines.prediction.adapters.large_embeddings.LargeEmbeddingsAdapter +import com.twitter.timelines.prediction.adapters.large_embeddings.OriginalTweetLargeEmbeddingsAdapter +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class OriginalTweetLargeEmbeddingsFeatureHydrator @Inject() ( + statsReceiver: StatsReceiver, + override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] + with HomeMixerLargeEmbeddingsFeatureHydrator { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("OriginalTweetLargeEmbeddings") + + override val features: Set[Feature[_, _]] = + Set(OriginalTweetLargeEmbeddingsFeature, OriginalTweetLargeEmbeddingsKeyFeature) + + override val adapter: LargeEmbeddingsAdapter = OriginalTweetLargeEmbeddingsAdapter + + override val cacheType: hmf.Cache = hmf.Cache.TweetLargeEmbeddings + + override val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableLargeEmbeddingsFeatureHydrationParam) + + // Hashing Features + override val defaultHashingFeatureParams: HashingFeatureParams = HashingFeatureParams( + scales = Seq(1131302000L, 303023026L), + biases = Seq(799473858L, 600426834L), + modulus = 3588720353L, + bucketSize = 10000000L, + ) + + override val modelName2HashingFeatureParams: Map[String, HashingFeatureParams] = Map( + "hr_video_prod__v3_realtime" -> HashingFeatureParams( + scales = Seq(1487022661L, 1399245971L), + biases = Seq(1372992088L, 632996194L), + modulus = 2865175829L, + bucketSize = 10000000L, + ), + "hr_video_prod__v2_lembeds" -> HashingFeatureParams( + scales = Seq(2516541900L, 2376187492L), + biases = Seq(3022238687L, 1571354734L), + modulus = 3047336911L, + bucketSize = 1000000L, + ), + "hr_prod__v4_embeds_230M" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + "hr_prod__v5_embeds_230M_and_transformer" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + "hr_prod__v5_watchtime" -> HashingFeatureParams( + scales = Seq(407033648L, 940305868L), + biases = Seq(494266171L, 269596788L), + modulus = 949146421L, + bucketSize = 100000000L, + ), + "hr_prod__v6_transformer_v2" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + "hr_prod__v6_mixed_training" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + "hr_prod__v6_transformer_v2_kafka_merge_join" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + "hr_prod__v6_transformer_v2_realtime_debias_21apr" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + "hr_video_prod__v4_realtime" -> HashingFeatureParams( + scales = Seq(1487022661L, 1399245971L), + biases = Seq(1372992088L, 632996194L), + modulus = 2865175829L, + bucketSize = 10000000L, + ), + "hr_video_prod__v4_realtime_mergehead" -> HashingFeatureParams( + scales = Seq(1487022661L, 1399245971L), + biases = Seq(1372992088L, 632996194L), + modulus = 2865175829L, + bucketSize = 10000000L, + ), + ) + + private val batchSize = 25 + + private def getBatchedFeatureMap( + modelName: String, + candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], + ): Future[Seq[FeatureMap]] = { + val originalTweetIds = + candidatesBatch.map { candidate => + getOriginalTweetId(candidate.candidate, candidate.features) + } + + getLargeEmbeddings(originalTweetIds, modelName).map { responses => + responses.map { largeEmbeddingResponse => + FeatureMapBuilder() + .add(OriginalTweetLargeEmbeddingsFeature, largeEmbeddingResponse.dataRecord) + .add(OriginalTweetLargeEmbeddingsKeyFeature, largeEmbeddingResponse.hashedKeys) + .build() + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val modelName = query.params(ModelNameParam) + OffloadFuturePools.offloadBatchSeqToFutureSeq( + candidates, + getBatchedFeatureMap(modelName, _), + batchSize + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala index b7aae811b..c8c330c9e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala @@ -4,11 +4,15 @@ import com.twitter.conversions.DurationOps._ import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedAuthorIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct +import com.twitter.home_mixer.param.HomeGlobalParams.ExcludeServedAuthorIdsDurationParam +import com.twitter.home_mixer.param.HomeGlobalParams.ExcludeServedTweetIdsDurationParam +import com.twitter.home_mixer.param.HomeGlobalParams.ExcludeServedTweetIdsNumberParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -24,6 +28,7 @@ import com.twitter.timelineservice.model.TimelineQuery import com.twitter.timelineservice.model.core.TimelineKind import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.util.Time + import javax.inject.Inject import javax.inject.Singleton @@ -39,17 +44,16 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() ( private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize") private val WhoToFollowExcludedUserIdsLimit = 1000 - private val ServedTweetIdsDuration = 10.minutes - private val ServedTweetIdsLimit = 100 private val ServedTweetPreviewIdsDuration = 10.hours private val ServedTweetPreviewIdsLimit = 10 - override val features: Set[Feature[_, _]] = - Set( - ServedTweetIdsFeature, - ServedTweetPreviewIdsFeature, - PersistenceEntriesFeature, - WhoToFollowExcludedUserIdsFeature) + override val features: Set[Feature[_, _]] = Set( + ServedTweetIdsFeature, + ServedTweetPreviewIdsFeature, + PersistenceEntriesFeature, + WhoToFollowExcludedUserIdsFeature, + ServedAuthorIdsFeature + ) private val supportedClients = Seq( ClientPlatform.IPhone, @@ -85,28 +89,47 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() ( clientAppId = query.clientContext.appId, userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString)) - val servedTweetIds = timelineResponses + val recentTimelineItems = timelineResponses .filter(_.clientPlatform == clientPlatform) - .filter(_.servedTime >= Time.now - ServedTweetIdsDuration) .sortBy(-_.servedTime.inMilliseconds) - .flatMap( - _.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit)) + + val servedTweetIds = recentTimelineItems + .filter(_.servedTime >= Time.now - query.params(ExcludeServedTweetIdsDurationParam)) + .flatMap(_.entries + .flatMap(_.tweetIds(includeSourceTweets = true)).take( + query.params(ExcludeServedTweetIdsNumberParam))) servedTweetIdsSizeStat.add(servedTweetIds.size) - val servedTweetPreviewIds = timelineResponses - .filter(_.clientPlatform == clientPlatform) + val servedTweetPreviewIds = recentTimelineItems .filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration) - .sortBy(-_.servedTime.inMilliseconds) - .flatMap(_.entries - .filter(_.entityIdType == EntityIdType.TweetPreview) - .flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit)) + .flatMap( + _.entries + .filter(_.entityIdType == EntityIdType.TweetPreview) + .flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit)) + + val servedAuthorIds: Map[Long, Seq[Long]] = recentTimelineItems + .filter(_.servedTime >= Time.now - query.params(ExcludeServedAuthorIdsDurationParam)) + .flatMap { timelineResponse => + timelineResponse.entries + .filter(_.entityIdType == EntityIdType.Tweet) + .flatMap { entry => + val authorId = entry.sourceAuthorIds.headOption.getOrElse(-1L) + if (authorId != -1L) { + // only include entries with valid author IDs + entry.tweetIds(includeSourceTweets = true).map(tweetId => (authorId, tweetId)) + } else Seq.empty + } + } + .groupBy(_._1) + .mapValues(_.map(_._2).distinct) FeatureMapBuilder() .add(ServedTweetIdsFeature, servedTweetIds) .add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds) .add(PersistenceEntriesFeature, timelineResponses) .add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds) + .add(ServedAuthorIdsFeature, servedAuthorIds) .build() } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PhoenixRescoringFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PhoenixRescoringFeatureHydrator.scala new file mode 100644 index 000000000..bddd403c7 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PhoenixRescoringFeatureHydrator.scala @@ -0,0 +1,48 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.PhoenixScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnablePhoenixScorerParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnablePhoenixRescoreParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnablePhoenixScoreParam +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object PhoenixRescoringFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("PhoenixRescoring") + + override val features: Set[Feature[_, _]] = Set(ScoreFeature) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + // Use rescoring to change scoreFeature only when scoring didn't happen using Phoenix + val usePhoenixRescoring = + query.params(EnablePhoenixScorerParam) && + query.params(EnablePhoenixRescoreParam) && + !query.params(EnablePhoenixScoreParam) + val finalScores = candidates.map { candidate => + val score = candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0) + val weightedModelScore = + candidate.features.getOrElse(WeightedModelScoreFeature, None).getOrElse(0.0) + val phoenixScore = candidate.features.getOrElse(PhoenixScoreFeature, None).getOrElse(0.0) + // The multiplier is for heuristics, it might not always be accurate for listwise heuristics + if (score == 0.0 | weightedModelScore == 0.0) 0.0 + else if (usePhoenixRescoring) phoenixScore * (score / weightedModelScore) + else score + } + Stitch.value(finalScores.map(score => FeatureMap(ScoreFeature, Some(score)))) + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PostContextFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PostContextFeatureHydrator.scala new file mode 100644 index 000000000..e3005cc29 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PostContextFeatureHydrator.scala @@ -0,0 +1,101 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.GenericPostContextFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.param.HomeGlobalParams.MaxPostContextDuplicatesPerRequest +import com.twitter.home_mixer.param.HomeGlobalParams.MaxPostContextPostsPerRequest +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.timelines.render.{thriftscala => urt} +import com.twitter.strato.catalog.Fetch +import com.twitter.strato.generated.client.events.urt.PostContextClientColumn +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Random + +@Singleton +class PostContextFeatureHydrator @Inject() ( + postContextClientColumn: PostContextClientColumn) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PostContext") + + override val features: Set[Feature[_, _]] = Set(GenericPostContextFeature) + + private val fetcher: Fetcher[Long, Unit, urt.GenericContext] = postContextClientColumn.fetcher + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val maxPosts = query.params(MaxPostContextPostsPerRequest) + val maxDuplicates = query.params(MaxPostContextDuplicatesPerRequest) + + val contextsStitch: Stitch[Seq[Option[urt.GenericContext]]] = Stitch.collect { + candidates.map { candidate => + val servedType = candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) + val isPromoted = { + servedType == hmt.ServedType.ForYouPromoted || servedType == hmt.ServedType.FollowingPromoted + } + + val isOriginal = (!candidate.features.getOrElse(IsRetweetFeature, false)) && + candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty + + if (isPromoted || !isOriginal) { + // Promoted tweets are not eligible for Post Context. + Stitch.value(None) + } else { + fetcher.fetch(candidate.candidate.id, ()).map { + case Fetch.Result(Some(postContext), _) => Some(postContext) + case _ => None + } + } + } + } + + contextsStitch.map { rawContexts => + val urlCount = scala.collection.mutable.Map.empty[String, Int] + val afterDupFilter: Vector[Option[urt.GenericContext]] = + rawContexts.map { + case Some(ctx) => + val url = ctx.url.url + val seen = urlCount.getOrElse(url, 0) + if (seen < maxDuplicates) { + urlCount.update(url, seen + 1) + Some(ctx) + } else None // drop: duplicate overflow + case None => None + }(collection.breakOut) + + val keptIndices: Vector[Int] = + afterDupFilter.iterator.zipWithIndex.collect { case (Some(_), idx) => idx }.toVector + + val indicesToDrop: Set[Int] = + if (keptIndices.size <= maxPosts) Set.empty + else { + val numToDrop = keptIndices.size - maxPosts + Random.shuffle(keptIndices).take(numToDrop).toSet + } + + afterDupFilter.zipWithIndex.map { + case (Some(ctx), idx) if !indicesToDrop.contains(idx) => + FeatureMapBuilder().add(GenericPostContextFeature, Some(ctx)).build() + + case _ => + FeatureMapBuilder().add(GenericPostContextFeature, None).build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RateLimitQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RateLimitQueryFeatureHydrator.scala new file mode 100644 index 000000000..90a646229 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RateLimitQueryFeatureHydrator.scala @@ -0,0 +1,54 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.ViewerIsRateLimited +import com.twitter.home_mixer.param.HomeGlobalParams.RateLimitTestIdsParam +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.limiter.{thriftscala => t} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class RateLimitQueryFeatureHydrator @Inject() (limiterClient: t.LimitService.MethodPerEndpoint) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RateLimit") + + override val features: Set[Feature[_, _]] = Set(ViewerIsRateLimited) + + val RegularFeature = "graphql_global_regular_tweets_read" + val NewFeature = "graphql_global_new_tweets_read" + val SoftFeature = "graphql_global_soft_user_tweets_read" + val SuspendedFeature = "graphql_global_suspended_user_tweets_read" + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getOptionalUserId + val appId = query.clientContext.appId + + val rateLimitTestIds = query.params(RateLimitTestIdsParam) + + val usagesStitch = Seq( + t.FeatureRequest(RegularFeature, userId, applicationId = appId), + t.FeatureRequest(NewFeature, userId, applicationId = appId), + t.FeatureRequest(SoftFeature, userId, applicationId = appId), + t.FeatureRequest(SuspendedFeature, userId, applicationId = appId) + ).map { request => Stitch.callFuture(limiterClient.getLimitUsage(None, Some(request))) } + + Stitch.collect(usagesStitch).map { usage => + val limited = + if (rateLimitTestIds.contains(userId.get)) + usage.map(u => u.remaining.toDouble / u.limit).exists(_ < 0.999) + else usage.map(_.remaining).exists(_ == 0) + + FeatureMapBuilder().add(ViewerIsRateLimited, limited).build() + } + } + + override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70)) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala index 92ab90f2c..0ebe03afd 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala @@ -1,7 +1,7 @@ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphInNetworkScores +import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphInNetworkScoresOnPrem import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder @@ -17,7 +17,7 @@ import javax.inject.Singleton @Singleton case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() ( - @Named(RealGraphInNetworkScores) store: ReadableStore[Long, Seq[wtf.Candidate]]) + @Named(RealGraphInNetworkScoresOnPrem) realGraphMhStore: ReadableStore[Long, Seq[wtf.Candidate]]) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = @@ -28,7 +28,8 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() ( private val RealGraphCandidateCount = 1000 override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - Stitch.callFuture(store.get(query.getRequiredUserId)).map { realGraphFollowedUsers => + + Stitch.callFuture(realGraphMhStore.get(query.getRequiredUserId)).map { realGraphFollowedUsers => val realGraphScoresFeatures = realGraphFollowedUsers .getOrElse(Seq.empty) .sortBy(-_.score) @@ -41,6 +42,7 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() ( } // Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97] + // v1 logic: src/scala/com/twitter/interaction_graph/scalding/jobs/scoring/InteractionGraphScoringJob.scala?L77-80 private def scaleScore(score: Double): Double = if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphQueryFeatureHydrator.scala new file mode 100644 index 000000000..f14451517 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphQueryFeatureHydrator.scala @@ -0,0 +1,57 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableOnPremRealGraphQueryFeatures +import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphFeatureRepository +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.repository.Repository +import com.twitter.stitch.Stitch +import com.twitter.timelines.model.UserId +import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphEdgeFeatures +import com.twitter.timelines.real_graph.{thriftscala => rg} +import com.twitter.user_session_store.{thriftscala => uss} + +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +object RealGraphFeatures extends Feature[PipelineQuery, Option[Map[UserId, RealGraphEdgeFeatures]]] + +@Singleton +class RealGraphQueryFeatureHydrator @Inject() ( + @Named(RealGraphFeatureRepository) repository: Repository[Long, Option[uss.UserSession]]) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("RealGraphFeatures") + + override val features: Set[Feature[_, _]] = Set(RealGraphFeatures) + + override def onlyIf( + query: PipelineQuery + ): Boolean = { + !query.params(EnableOnPremRealGraphQueryFeatures) + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + Stitch.callFuture { + repository(query.getRequiredUserId).map { userSession => + val realGraphFeaturesMap = userSession.flatMap { userSession => + userSession.realGraphFeatures.collect { + case rg.RealGraphFeatures.V1(realGraphFeatures) => + val edgeFeatures = realGraphFeatures.edgeFeatures ++ realGraphFeatures.oonEdgeFeatures + edgeFeatures.map { edge => edge.destId -> edge }.toMap + } + } + + FeatureMapBuilder().add(RealGraphFeatures, realGraphFeaturesMap).build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala new file mode 100644 index 000000000..83289e756 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala @@ -0,0 +1,135 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature +import com.twitter.home_mixer.util.MissingKeyException +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter +import com.twitter.timelines.prediction.adapters.real_graph.RealGraphFeaturesAdapter +import com.twitter.timelines.real_graph.v1.{thriftscala => v1} +import com.twitter.timelines.real_graph.{thriftscala => rg} +import com.twitter.util.Throw +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object RealGraphViewerAuthorDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +object RealGraphViewerAuthorsDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class RealGraphViewerAuthorFeatureHydrator @Inject() () + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("RealGraphViewerAuthor") + + override val features: Set[Feature[_, _]] = + Set(RealGraphViewerAuthorDataRecordFeature, RealGraphViewerAuthorsDataRecordFeature) + + private val realGraphEdgeFeaturesAdapter = new RealGraphFeaturesAdapter + private val realGraphEdgeFeaturesCombineAdapter = + new RealGraphEdgeFeaturesCombineAdapter(prefix = "authors.realgraph") + + private val MissingKeyFeatureMap = FeatureMapBuilder() + .add(RealGraphViewerAuthorDataRecordFeature, Throw(MissingKeyException)) + .add(RealGraphViewerAuthorsDataRecordFeature, Throw(MissingKeyException)) + .build() + + private val batchSize = 64 + + def getFeatureMap( + candidate: CandidateWithFeatures[TweetCandidate], + viewerId: Long, + realGraphFeatures: Map[Long, v1.RealGraphEdgeFeatures] + ): FeatureMap = + candidate.features.getOrElse(AuthorIdFeature, None) match { + case Some(authorId) => + val realGraphAuthorFeatures = + getRealGraphViewerAuthorFeatures(viewerId, authorId, realGraphFeatures) + val realGraphAuthorDataRecord = realGraphEdgeFeaturesAdapter + .adaptToDataRecords(realGraphAuthorFeatures).asScala.headOption.getOrElse(new DataRecord) + + val combinedRealGraphFeaturesDataRecord = for { + inReplyToAuthorId <- candidate.features.getOrElse(InReplyToUserIdFeature, None) + } yield { + val combinedRealGraphFeatures = + getCombinedRealGraphFeatures(Seq(authorId, inReplyToAuthorId), realGraphFeatures) + realGraphEdgeFeaturesCombineAdapter + .adaptToDataRecords(Some(combinedRealGraphFeatures)).asScala.headOption + .getOrElse(new DataRecord) + } + + FeatureMap( + RealGraphViewerAuthorDataRecordFeature, + realGraphAuthorDataRecord, + RealGraphViewerAuthorsDataRecordFeature, + combinedRealGraphFeaturesDataRecord.getOrElse(new DataRecord) + ) + case _ => MissingKeyFeatureMap + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val viewerId = query.getRequiredUserId + val realGraphFeatures = query.features + .flatMap(_.getOrElse(RealGraphFeatures, None)) + .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) + + OffloadFuturePools.offloadBatchElementToElement( + candidates, + getFeatureMap(_, viewerId, realGraphFeatures), + batchSize) + } + + private def getRealGraphViewerAuthorFeatures( + viewerId: Long, + authorId: Long, + realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] + ): rg.UserRealGraphFeatures = { + realGraphEdgeFeaturesMap.get(authorId) match { + case Some(realGraphEdgeFeatures) => + rg.UserRealGraphFeatures( + srcId = viewerId, + features = rg.RealGraphFeatures.V1( + v1.RealGraphFeatures(edgeFeatures = Seq(realGraphEdgeFeatures)))) + case _ => + rg.UserRealGraphFeatures( + srcId = viewerId, + features = rg.RealGraphFeatures.V1(v1.RealGraphFeatures(edgeFeatures = Seq.empty))) + } + } +} + +object RealGraphViewerAuthorFeatureHydrator { + def getCombinedRealGraphFeatures( + userIds: Seq[Long], + realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] + ): rg.RealGraphFeatures = { + val edgeFeatures = userIds.flatMap(realGraphEdgeFeaturesMap.get) + rg.RealGraphFeatures.V1(v1.RealGraphFeatures(edgeFeatures = edgeFeatures)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala new file mode 100644 index 000000000..d3e61a818 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala @@ -0,0 +1,94 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableRealGraphViewerRelatedUsersFeaturesParam +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter +import com.twitter.timelines.real_graph.v1.{thriftscala => v1} +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object RealGraphViewerRelatedUsersDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class RealGraphViewerRelatedUsersFeatureHydrator @Inject() () + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] + with WithDefaultFeatureMap { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("RealGraphViewerRelatedUsers") + + override val features: Set[Feature[_, _]] = Set(RealGraphViewerRelatedUsersDataRecordFeature) + + override val defaultFeatureMap: FeatureMap = FeatureMap( + RealGraphViewerRelatedUsersDataRecordFeature, + RealGraphViewerRelatedUsersDataRecordFeature.defaultValue) + + private val RealGraphEdgeFeaturesCombineAdapter = new RealGraphEdgeFeaturesCombineAdapter + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableRealGraphViewerRelatedUsersFeaturesParam) + + val batchSize = 64 + + def getFeatureMap( + candidate: CandidateWithFeatures[TweetCandidate], + realGraphQueryFeatures: Map[Long, v1.RealGraphEdgeFeatures] + ): FeatureMap = { + val allRelatedUserIds = getRelatedUserIds(candidate.features) + val realGraphFeatures = + RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures( + allRelatedUserIds, + realGraphQueryFeatures + ) + val realGraphFeaturesDataRecord = RealGraphEdgeFeaturesCombineAdapter + .adaptToDataRecords(Some(realGraphFeatures)).asScala.headOption + .getOrElse(new DataRecord) + + FeatureMap(RealGraphViewerRelatedUsersDataRecordFeature, realGraphFeaturesDataRecord) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val realGraphQueryFeatures = query.features + .flatMap(_.getOrElse(RealGraphFeatures, None)) + .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) + + OffloadFuturePools.offloadBatchElementToElement( + candidates, + getFeatureMap(_, realGraphQueryFeatures), + batchSize) + } + + private def getRelatedUserIds(features: FeatureMap): Seq[Long] = { + (CandidatesUtil.getEngagerUserIds(features) ++ + features.getOrElse(AuthorIdFeature, None) ++ + features.getOrElse(MentionUserIdFeature, Seq.empty) ++ + features.getOrElse(SourceUserIdFeature, None) ++ + features.getOrElse(DirectedAtUserIdFeature, None)).distinct + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeEntityRealGraphQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeEntityRealGraphQueryFeatureHydrator.scala new file mode 100644 index 000000000..3ab3bfb3b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeEntityRealGraphQueryFeatureHydrator.scala @@ -0,0 +1,52 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.EntityRealGraphClientStore +import com.twitter.home_mixer.util.ObservedKeyValueResultHandler +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.recos.entities.{thriftscala => ent} +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.wtf.entity_real_graph.{thriftscala => erg} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +object RealTimeEntityRealGraphFeatures + extends Feature[PipelineQuery, Option[Map[ent.Entity, Map[erg.EngagementType, erg.Features]]]] + +@Singleton +class RealTimeEntityRealGraphQueryFeatureHydrator @Inject() ( + @Named(EntityRealGraphClientStore) client: ReadableStore[ + erg.EntityRealGraphRequest, + erg.EntityRealGraphResponse + ], + override val statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with ObservedKeyValueResultHandler { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("RealTimeEntityRealGraphFeatures") + + override val features: Set[Feature[_, _]] = Set(RealTimeEntityRealGraphFeatures) + + override val statScope: String = identifier.toString + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val entityRealGraphRequest = erg.EntityRealGraphRequest( + userId = query.getRequiredUserId, + entityTypes = Set(erg.EntityType.SemanticCore), + normalizeCounts = Some(true)) + Stitch.callFuture { + client.get(entityRealGraphRequest).map { response => + val engagements = response.map(_.response.mapValues(_.toMap).toMap) + FeatureMapBuilder().add(RealTimeEntityRealGraphFeatures, engagements).build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala new file mode 100644 index 000000000..cf3217136 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala @@ -0,0 +1,60 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.adapters.realtime_interaction_graph.RealTimeInteractionGraphFeaturesAdapter +import com.twitter.timelines.prediction.features.realtime_interaction_graph.RealTimeInteractionGraphEdgeFeatures +import com.twitter.util.Time +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object RealTimeInteractionGraphEdgeFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() () + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("RealTimeInteractionGraphEdge") + + override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphEdgeFeature) + + private val realTimeInteractionGraphFeaturesAdapter = new RealTimeInteractionGraphFeaturesAdapter + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { + val userVertex = + query.features.flatMap(_.getOrElse(RealTimeInteractionGraphUserVertexQueryFeature, None)) + val realTimeInteractionGraphFeaturesMap = + userVertex.map(RealTimeInteractionGraphEdgeFeatures(_, Time.now)) + + candidates.map { candidate => + val feature = candidate.features.getOrElse(AuthorIdFeature, None).flatMap { authorId => + realTimeInteractionGraphFeaturesMap.flatMap(_.get(authorId)) + } + + val dataRecordFeature = + realTimeInteractionGraphFeaturesAdapter.adaptToDataRecords(feature).asScala.head + + FeatureMap(RealTimeInteractionGraphEdgeFeature, dataRecordFeature) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala new file mode 100644 index 000000000..76f8b7782 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala @@ -0,0 +1,48 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.google.inject.name.Named +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexCache +import com.twitter.home_mixer.util.ObservedKeyValueResultHandler +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.cache.ReadCache +import com.twitter.stitch.Stitch +import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig} +import javax.inject.Inject +import javax.inject.Singleton + +object RealTimeInteractionGraphUserVertexQueryFeature + extends Feature[PipelineQuery, Option[ig.UserVertex]] + +@Singleton +class RealTimeInteractionGraphUserVertexQueryFeatureHydrator @Inject() ( + @Named(RealTimeInteractionGraphUserVertexCache) client: ReadCache[Long, ig.UserVertex], + override val statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with ObservedKeyValueResultHandler { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("RealTimeInteractionGraphUserVertex") + + override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphUserVertexQueryFeature) + + override val statScope: String = identifier.toString + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + + Stitch.callFuture( + client.get(Seq(userId)).map { results => + val feature = observedGet(key = Some(userId), keyValueResult = results) + FeatureMapBuilder() + .add(RealTimeInteractionGraphUserVertexQueryFeature, feature) + .build() + } + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala index 3968c9532..506b2adea 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala @@ -36,6 +36,7 @@ class RequestQueryFeatureHydrator[ override val features: Set[Feature[_, _]] = Set( AccountAgeFeature, ClientIdFeature, + DeviceCountryFeature, DeviceLanguageFeature, GetInitialFeature, GetMiddleFeature, @@ -48,7 +49,7 @@ class RequestQueryFeatureHydrator[ PollingFeature, PullToRefreshFeature, RequestJoinIdFeature, - ServedRequestIdFeature, + ServedIdFeature, TimestampFeature, TimestampGMTDowFeature, TimestampGMTHourFeature, @@ -63,8 +64,8 @@ class RequestQueryFeatureHydrator[ private def getLanguageISOFormatByCode(languageCode: String): String = ThriftLanguageUtil.getLanguageCodeOf(ThriftLanguageUtil.getThriftLanguageOf(languageCode)) - private def getRequestJoinId(servedRequestId: Long): Option[Long] = - Some(RequestJoinKeyContext.current.flatMap(_.requestJoinId).getOrElse(servedRequestId)) + private def getRequestJoinId: Option[Long] = + RequestJoinKeyContext.current.flatMap(_.requestJoinId) private def hasDarkRequest: Option[Boolean] = ForwardAnnotation.current .getOrElse(Seq[BinaryAnnotation]()) @@ -73,12 +74,13 @@ class RequestQueryFeatureHydrator[ override def hydrate(query: Query): Stitch[FeatureMap] = { val requestContext = query.deviceContext.flatMap(_.requestContextValue) - val servedRequestId = UUID.randomUUID.getMostSignificantBits + val servedId = UUID.randomUUID.getMostSignificantBits val timestamp = query.queryTime.inMilliseconds val featureMap = FeatureMapBuilder() .add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt)) .add(ClientIdFeature, query.clientContext.appId) + .add(DeviceCountryFeature, query.getCountryCode) .add(DeviceLanguageFeature, query.getLanguageCode.map(getLanguageISOFormatByCode)) .add( GetInitialFeature, @@ -103,8 +105,8 @@ class RequestQueryFeatureHydrator[ .add(IsLaunchRequestFeature, requestContext.contains(RequestContext.Launch)) .add(PollingFeature, query.deviceContext.exists(_.isPolling.contains(true))) .add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh)) - .add(ServedRequestIdFeature, Some(servedRequestId)) - .add(RequestJoinIdFeature, getRequestJoinId(servedRequestId)) + .add(ServedIdFeature, Some(servedId)) + .add(RequestJoinIdFeature, getRequestJoinId) .add(TimestampFeature, timestamp) .add(TimestampGMTDowFeature, dowFromTimestamp(timestamp)) .add(TimestampGMTHourFeature, hourFromTimestamp(timestamp)) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestTimeQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestTimeQueryFeatureHydrator.scala new file mode 100644 index 000000000..9e9efdbdd --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestTimeQueryFeatureHydrator.scala @@ -0,0 +1,122 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature +import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature +import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.util.FDsl._ +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.ACCOUNT_AGE_INTERVAL +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_12_MONTH_NEW_USER +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_30_DAY_NEW_USER +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_BETWEEN_NON_POLLING_REQUESTS_AVG +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_LAST_NON_POLLING_REQUEST +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.USER_ID_IS_SNOWFLAKE_ID +import com.twitter.user_session_store.ReadRequest +import com.twitter.user_session_store.ReadWriteUserSessionStore +import com.twitter.user_session_store.UserSessionDataset +import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset +import com.twitter.util.Time +import javax.inject.Inject +import javax.inject.Singleton + +object RequestTimeDataRecordFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +case class RequestTimeQueryFeatureHydrator @Inject() ( + userSessionStore: ReadWriteUserSessionStore) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RequestTime") + + override val features: Set[Feature[_, _]] = Set( + FollowingLastNonPollingTimeFeature, + LastNonPollingTimeFeature, + NonPollingTimesFeature, + RequestTimeDataRecordFeature + ) + + private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.NonPollingTimes) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + userSessionStore + .read(ReadRequest(query.getRequiredUserId, datasets)) + .map { userSession => + val nonPollingTimestamps = userSession.flatMap(_.nonPollingTimestamps) + + val lastNonPollingTime = nonPollingTimestamps + .flatMap(_.nonPollingTimestampsMs.headOption) + .map(Time.fromMilliseconds) + + val followingLastNonPollingTime = nonPollingTimestamps + .flatMap(_.mostRecentHomeLatestNonPollingTimestampMs) + .map(Time.fromMilliseconds) + + val nonPollingTimes = nonPollingTimestamps + .map(_.nonPollingTimestampsMs) + .getOrElse(Seq.empty) + + val requestTimeDataRecord = getRequestTimeDataRecord(query, nonPollingTimes) + + FeatureMapBuilder() + .add(FollowingLastNonPollingTimeFeature, followingLastNonPollingTime) + .add(LastNonPollingTimeFeature, lastNonPollingTime) + .add(NonPollingTimesFeature, nonPollingTimes) + .add(RequestTimeDataRecordFeature, requestTimeDataRecord) + .build() + } + } + + def getRequestTimeDataRecord(query: PipelineQuery, nonPollingTimes: Seq[Long]): DataRecord = { + val requestTimeMs = query.queryTime.inMillis + val accountAge = SnowflakeId.timeFromIdOpt(query.getRequiredUserId) + val timeSinceAccountCreation = accountAge.map(query.queryTime.since) + val timeSinceEarliestNonPollingRequest = + nonPollingTimes.lastOption.map(requestTimeMs - _) + val timeSinceLastNonPollingRequest = + nonPollingTimes.headOption.map(requestTimeMs - _) + + new DataRecord() + .setFeatureValue(USER_ID_IS_SNOWFLAKE_ID, accountAge.isDefined) + .setFeatureValue( + IS_30_DAY_NEW_USER, + timeSinceAccountCreation.map(_ < 30.days).getOrElse(false) + ) + .setFeatureValue( + IS_12_MONTH_NEW_USER, + timeSinceAccountCreation.map(_ < 365.days).getOrElse(false) + ) + .setFeatureValueFromOption( + ACCOUNT_AGE_INTERVAL, + timeSinceAccountCreation.flatMap(AccountAgeInterval.fromDuration).map(_.id.toLong) + ) + .setFeatureValueFromOption( + TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS, + timeSinceAccountCreation.map(_.inSeconds.toDouble) + ) + .setFeatureValueFromOption( + TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, + timeSinceEarliestNonPollingRequest.map(_.toDouble / math.max(1.0, nonPollingTimes.size)) + ) + .setFeatureValueFromOption( + TIME_SINCE_LAST_NON_POLLING_REQUEST, + timeSinceLastNonPollingRequest.map(_.toDouble) + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala index 92c351ecf..a7c9a27cf 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala @@ -27,7 +27,8 @@ import javax.inject.Singleton @Singleton class SGSValidSocialContextFeatureHydrator @Inject() ( socialGraph: SocialGraph) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with WithDefaultFeatureMap { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SGSValidSocialContext") @@ -37,6 +38,13 @@ class SGSValidSocialContextFeatureHydrator @Inject() ( SGSValidLikedByUserIdsFeature ) + override val defaultFeatureMap: FeatureMap = FeatureMap( + SGSValidFollowedByUserIdsFeature, + Seq.empty, + SGSValidLikedByUserIdsFeature, + Seq.empty + ) + private val MaxCountUsers = 10 override def apply( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala new file mode 100644 index 000000000..8a6cf3671 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala @@ -0,0 +1,82 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableSimClustersSimilarityFeaturesDeciderParam +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient +import com.twitter.timelines.configapi.decider.BooleanDeciderParam +import com.twitter.timelines.prediction.adapters.twistly.SimClustersRecentEngagementSimilarityFeaturesAdapter +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton + +object SimClustersEngagementSimilarityFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class SimClustersEngagementSimilarityFeatureHydrator @Inject() ( + simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("SimClustersEngagementSimilarity") + + override val features: Set[Feature[_, _]] = Set(SimClustersEngagementSimilarityFeature) + + private val simClustersRecentEngagementSimilarityFeaturesAdapter = + new SimClustersRecentEngagementSimilarityFeaturesAdapter + + override def onlyIf(query: PipelineQuery): Boolean = { + val param: BooleanDeciderParam = EnableSimClustersSimilarityFeaturesDeciderParam + query.params.apply(param) + } + + def getFeatureMaps( + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + userId: Long + ): Future[Seq[FeatureMap]] = { + val tweetToCandidates = + candidates.map(candidate => candidate.candidate.id -> candidate).toMap + val tweetIds = tweetToCandidates.keySet.toSeq + val userTweetEdges = tweetIds.map(tweetId => (userId, tweetId)) + + simClustersEngagementSimilarityClient + .getSimClustersRecentEngagementSimilarityScores(userTweetEdges).map { + simClustersRecentEngagementSimilarityScoresMap => + candidates.map { candidate => + val similarityFeatureOpt = simClustersRecentEngagementSimilarityScoresMap + .get(userId -> candidate.candidate.id).flatten + val dataRecordOpt = similarityFeatureOpt.map { similarityFeature => + simClustersRecentEngagementSimilarityFeaturesAdapter + .adaptToDataRecords(similarityFeature) + .get(0) + } + FeatureMap( + SimClustersEngagementSimilarityFeature, + dataRecordOpt.getOrElse(new DataRecord)) + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = + Stitch.callFuture { + getFeatureMaps(candidates, query.getRequiredUserId) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersLogFavBasedTweetFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersLogFavBasedTweetFeatureHydrator.scala new file mode 100644 index 000000000..36d97bfbc --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersLogFavBasedTweetFeatureHydrator.scala @@ -0,0 +1,97 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.DiversityRescoringFeatureHydrator.EmptyDataRecord +import com.twitter.home_mixer_features.thriftjava.HomeMixerFeaturesRequest +import com.twitter.home_mixer_features.{thriftjava => t} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object SimClustersLogFavBasedTweetFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class SimClustersLogFavBasedTweetFeatureHydrator @Inject() ( + homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "SimClustersLogFavBasedTweet") + + override val features: Set[Feature[_, _]] = Set(SimClustersLogFavBasedTweetFeature) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyTotalCounter = scopedStatsReceiver.counter("key/total") + + private val batchSize = 50 + + private def getEmbeddingsFromHMF( + tweetIds: Seq[Long] + ): Future[Seq[DataRecord]] = { + val keysSerialized = tweetIds.map(_.toString) + val request = new HomeMixerFeaturesRequest() + request.setKeys(keysSerialized.asJava) + request.setCache(t.Cache.LOG_FAV_BASED_TWEET_20M145K2020_EMBEDDINGS) + homeMixerFeatureService + .getHomeMixerFeatures(request) + .map { resp => unmarshallHomeMixerFeaturesResponse(resp) } + } + + private def unmarshallHomeMixerFeaturesResponse( + response: t.HomeMixerFeaturesResponse + ): Seq[DataRecord] = { + response.getHomeMixerFeatures.asScala.map { homeMixerFeatureOpt => + if (homeMixerFeatureOpt.isSetHomeMixerFeaturesType) { + val homeMixerFeature = homeMixerFeatureOpt.getHomeMixerFeaturesType + if (homeMixerFeature.isSet(t.HomeMixerFeaturesType._Fields.DATA_RECORD)) { + homeMixerFeature.getDataRecord + } else { + throw new Exception("Unexpected type") + } + } else EmptyDataRecord + } + } + + private def getBatchedFeatureMap( + candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]] + ): Future[Seq[FeatureMap]] = { + val tweetIds = + candidatesBatch.map { candidate => + keyTotalCounter.incr() + candidate.candidate.id + } + + getEmbeddingsFromHMF(tweetIds).map { response => + response.map { dataRecordOpt => + keyFoundCounter.incr() + FeatureMap(SimClustersLogFavBasedTweetFeature, dataRecordOpt) + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + OffloadFuturePools.offloadBatchSeqToFutureSeq(candidates, getBatchedFeatureMap, batchSize) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersUserSparseEmbeddingsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersUserSparseEmbeddingsQueryFeatureHydrator.scala new file mode 100644 index 000000000..31263d7e5 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersUserSparseEmbeddingsQueryFeatureHydrator.scala @@ -0,0 +1,82 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.dal.personal_data.{thriftjava => pd} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature +import com.twitter.product_mixer.core.feature.datarecord.SparseContinuousDataRecordCompatible +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.recommendations.simclusters_v2.InterestedIn20M145K2020OnUserClientColumn +import com.twitter.timelines.prediction.features.simcluster.SimclusterFeatures +import javax.inject.Inject +import javax.inject.Singleton + +object SimClustersUserLogFavSparseEmbeddingsDataRecordFeature + extends DataRecordOptionalFeature[PipelineQuery, Map[String, Double]] + with SparseContinuousDataRecordCompatible { + override val featureName: String = + SimclusterFeatures.SIMCLUSTER_USER_LOG_FAV_CLUSTER_SCORES.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredInterests) +} + +object SimClustersUserFollowSparseEmbeddingsDataRecordFeature + extends DataRecordOptionalFeature[PipelineQuery, Map[String, Double]] + with SparseContinuousDataRecordCompatible { + override val featureName: String = + SimclusterFeatures.SIMCLUSTER_USER_FOLLOW_CLUSTER_SCORES.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredInterests) +} + +@Singleton +class SimClustersUserSparseEmbeddingsQueryFeatureHydrator @Inject() ( + interestedIn20M145K2020OnUserClientColumn: InterestedIn20M145K2020OnUserClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("SimClustersUserSparseEmbeddingsQuery") + + override val features: Set[Feature[_, _]] = + Set( + SimClustersUserLogFavSparseEmbeddingsDataRecordFeature, + SimClustersUserFollowSparseEmbeddingsDataRecordFeature) + + private val DefaultFeatureMap = FeatureMapBuilder() + .add(SimClustersUserLogFavSparseEmbeddingsDataRecordFeature, None) + .add(SimClustersUserFollowSparseEmbeddingsDataRecordFeature, None) + .build() + + override def hydrate( + query: PipelineQuery, + ): Stitch[FeatureMap] = { + val interestedInEmbeddingsOptStitch = interestedIn20M145K2020OnUserClientColumn.fetcher + .fetch(query.getRequiredUserId) + .map { result => + result.v.map { interestedInEmbeddings => + interestedInEmbeddings.clusterIdToScores + } + } + interestedInEmbeddingsOptStitch + .map { interestedInEmbeddingsOpt => + val logFavEmbeddings = interestedInEmbeddingsOpt.map { interestedInEmbeddings => + interestedInEmbeddings.map { + case (key, value) => (key.toString, value.logFavScore.getOrElse(0.0)) + }.toMap + } + val followEmbeddings = interestedInEmbeddingsOpt.map { interestedInEmbeddings => + interestedInEmbeddings.map { + case (key, value) => (key.toString, value.followScore.getOrElse(0.0)) + }.toMap + } + FeatureMapBuilder() + .add(SimClustersUserLogFavSparseEmbeddingsDataRecordFeature, logFavEmbeddings) + .add(SimClustersUserFollowSparseEmbeddingsDataRecordFeature, followEmbeddings) + .build() + }.handle { case _ => DefaultFeatureMap } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersUserTweetScoresHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersUserTweetScoresHydrator.scala new file mode 100644 index 000000000..c6a55a500 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersUserTweetScoresHydrator.scala @@ -0,0 +1,97 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.dal.personal_data.{thriftjava => pd} +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature +import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.catalog.Fetch +import com.twitter.strato.generated.client.ml.featureStore.SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton + +object SimClustersUserInterestedInTweetEmbeddingDataRecordFeature + extends DataRecordOptionalFeature[TweetCandidate, Double] + with DoubleDataRecordCompatible { + override val featureName: String = + "user-tweet.recommendations.sim_clusters_scores.user_interested_in_tweet_embedding_dot_product_20m_145k_2020" + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredInterests) +} + +@Singleton +class SimClustersUserTweetScoresHydrator @Inject() ( + simClustersColumn: SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn, + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("SimClustersUserTweetScores") + + override val features: Set[Feature[_, _]] = + Set(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") + private val keySkipCounter = scopedStatsReceiver.counter("key/skip") + + private val DefaultFeatureMap = FeatureMapBuilder() + .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, None) + .build() + private val MinFavToHydrate = 9 + private val batchSize = 64 + + def getFeatureMaps( + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + userId: Long + ): Future[Seq[FeatureMap]] = { + val featureMapStitch = Stitch.traverse(candidates) { candidate => + val ebFeatures = candidate.features.getOrElse(EarlybirdFeature, None) + val favCount = ebFeatures.flatMap(_.favCountV2).getOrElse(0) + + if (ebFeatures.isEmpty || favCount >= MinFavToHydrate) { + simClustersColumn.fetcher + .fetch((userId, candidate.candidate.id), Unit) + .map { + case Fetch.Result(response, _) => + if (response.nonEmpty) keyFoundCounter.incr() + else keyNotFoundCounter.incr() + FeatureMapBuilder() + .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, response) + .build() + case _ => + keyFailureCounter.incr() + DefaultFeatureMap + } + } else { + keySkipCounter.incr() + Stitch.value(DefaultFeatureMap) + } + } + Stitch.run(featureMapStitch) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + OffloadFuturePools.offloadBatchSeqToFutureSeq( + candidates, + getFeatureMaps(_, query.getRequiredUserId), + batchSize, + offload = true) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimclusterBasedTopAuthorsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimclusterBasedTopAuthorsQueryFeatureHydrator.scala new file mode 100644 index 000000000..8782b13b8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimclusterBasedTopAuthorsQueryFeatureHydrator.scala @@ -0,0 +1,111 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.cache.ExpiringLruInProcessCache +import com.twitter.servo.cache.InProcessCache +import com.twitter.simclusters_v2.thriftscala.ClusterDetails +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Random + +object SimclustersFavBasedTopAuthors extends Feature[PipelineQuery, Seq[(Long, Double)]] +object SimclustersFollowBasedTopAuthors extends Feature[PipelineQuery, Seq[(Long, Double)]] + +object SimclusterBasedTopAuthorsQueryFeatureHydrator { + private val BaseTTL = 60 * 24 + private val TTL = (BaseTTL + Random.nextInt(60)).minutes + + val cache: InProcessCache[String, Seq[(Long, Double)]] = + new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 150 * 1000) +} + +@Singleton +class SimclusterBasedTopAuthorsQueryFeatureHydrator @Inject() ( + store: ReadableStore[String, ClusterDetails]) + extends QueryFeatureHydrator[PipelineQuery] { + + import SimclusterBasedTopAuthorsQueryFeatureHydrator._ + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("SimclusterBasedTopAuthors") + + override val features: Set[Feature[_, _]] = + Set(SimclustersFavBasedTopAuthors, SimclustersFollowBasedTopAuthors) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val favBasedEmbeddings = + query.features + .flatMap(_.getOrElse(SimClustersUserLogFavSparseEmbeddingsDataRecordFeature, None)) + .getOrElse(Map.empty[String, Double]) + val followBasedEmbeddings = + query.features + .flatMap(_.getOrElse(SimClustersUserFollowSparseEmbeddingsDataRecordFeature, None)) + .getOrElse(Map.empty[String, Double]) + + val favBasedTopAuthorsStitch = Stitch.callFuture(getTopAuthorsWithScores(favBasedEmbeddings)) + val followBasedTopAuthorsStitch = + Stitch.callFuture(getTopAuthorsWithScores(followBasedEmbeddings)) + Stitch.join(favBasedTopAuthorsStitch, followBasedTopAuthorsStitch).map { + case (favBasedTopAuthors, followBasedTopAuthors) => + FeatureMap( + SimclustersFavBasedTopAuthors, + favBasedTopAuthors, + SimclustersFollowBasedTopAuthors, + followBasedTopAuthors + ) + } + } + + private def getTopAuthorsWithScores( + embeddings: Map[String, Double] + ): Future[Seq[(Long, Double)]] = { + val flattenedAuthorsWithScoresFut = Future + .collect { + embeddings.map { + case (clusterId, seedScore) => + getTopAuthorsWithScoresForCluster(clusterId).map { topAuthors => + topAuthors.map { + case (author, score) => (author, score * seedScore) + } + } + }.toSeq + }.map(_.flatten) + flattenedAuthorsWithScoresFut.map { flattenedAuthorsWithScores => + val authorsWithScores = + flattenedAuthorsWithScores.groupBy(_._1).mapValues(_.map(_._2).sum).toSeq + authorsWithScores.sortBy(-_._2) + } + } + + private def getTopAuthorsWithScoresForCluster( + clusterId: String + ): Future[Seq[(Long, Double)]] = { + cache + .get(clusterId) + .map(Future.value(_)) + .getOrElse { + store + .get(clusterId).map { clusterDetailsOpt => + val authorsWithScores = clusterDetailsOpt + .flatMap { clusterDetails => + clusterDetails.knownForUsersAndScores.map { knownForUsersAndScores => + knownForUsersAndScores.map { userAndScore => + (userAndScore.userId, userAndScore.score) + } + } + }.getOrElse(Seq.empty) + cache.set(clusterId, authorsWithScores) + authorsWithScores + }.handle { case _ => Seq.empty } + } + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SlopAuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SlopAuthorFeatureHydrator.scala new file mode 100644 index 000000000..b9f04a8fd --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SlopAuthorFeatureHydrator.scala @@ -0,0 +1,80 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.abuse.detection.scoring.{thriftscala => t} +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature +import com.twitter.home_mixer.model.HomeFeatures.SlopAuthorFeature +import com.twitter.home_mixer.model.HomeFeatures.SlopAuthorScoreFeature +import com.twitter.home_mixer.param.HomeGlobalParams.SlopMaxScore +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.hss.user_scores.api.HealthModelScoresOnUserClientColumn + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SlopAuthorFeatureHydrator @Inject() ( + healthModelScoresOnUserClientColumn: HealthModelScoresOnUserClientColumn) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SlopAuthor") + + override val features: Set[Feature[_, _]] = + Set(SlopAuthorFeature, DebugStringFeature, SlopAuthorScoreFeature) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct + val slopThreshold = query.params(SlopMaxScore) + Stitch + .collectToTry { + authorIds.map { authorId => + healthModelScoresOnUserClientColumn.fetcher + .fetch(authorId, Seq(t.UserHealthModel.NsfwConsumerFollowerScore)) + .map { response => + authorId -> response.v.flatMap { scoresMap => + scoresMap.get(t.UserHealthModel.NsfwConsumerFollowerScore) + } + } + } + }.map { authorNsfwScores => + val authorIdsToNsfwScoresMap = authorNsfwScores.flatMap(_.toOption).toMap + candidates.map { candidate => + val debugStringFeature = + candidate.features.getOrElse(DebugStringFeature, None).getOrElse("") + candidate.features.getOrElse(AuthorIdFeature, None) match { + case Some(authorId) => + val slopAuthorScore = + authorIdsToNsfwScoresMap.getOrElse(authorId, None).getOrElse(0.0) + FeatureMap( + SlopAuthorFeature, + slopAuthorScore > slopThreshold, + DebugStringFeature, + Some("%s Slop %.3f".format(debugStringFeature, slopAuthorScore)), + SlopAuthorScoreFeature, + Some(slopAuthorScore) + ) + case _ => + FeatureMap( + SlopAuthorFeature, + false, + DebugStringFeature, + Some(debugStringFeature), + SlopAuthorScoreFeature, + None + ) + } + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SpaceStateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SpaceStateFeatureHydrator.scala new file mode 100644 index 000000000..7f817a61f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SpaceStateFeatureHydrator.scala @@ -0,0 +1,78 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature +import com.twitter.periscope.api.{thriftscala => ps} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.catalog.Fetch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.periscope.CoreOnAudioSpaceClientColumn +import com.twitter.ubs.{thriftscala => ubs} +import javax.inject.Inject +import javax.inject.Singleton + +object SpaceStateFeature extends Feature[TweetCandidate, Option[ubs.BroadcastState]] + +@Singleton +class SpaceStateFeatureHydrator @Inject() ( + coreOnAudioSpaceClientColumn: CoreOnAudioSpaceClientColumn) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with WithDefaultFeatureMap { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SpaceState") + + override val features: Set[Feature[_, _]] = Set(SpaceStateFeature) + + private val pattern = """https?://(?:x|twitter).com/i/spaces/(\w+).*""".r + + private val fetcher: Fetcher[String, ps.AudioSpacesLookupContext, ubs.AudioSpace] = + coreOnAudioSpaceClientColumn.fetcher + + private val lookupContext = ps.AudioSpacesLookupContext(participantHydrationLevel = + Some(ps.ParticipantHydrationLevel.NoParticipantInfo)) + + override val defaultFeatureMap: FeatureMap = FeatureMap(SpaceStateFeature, None) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = + OffloadFuturePools.offloadStitch { + val spaceIdMap = candidates.flatMap { candidate => + candidate.features + .getOrElse(TweetUrlsFeature, Seq.empty) + .collectFirst { case pattern(spaceId) => spaceId } + .map(spaceId => candidate.candidate.id -> spaceId) + }.toMap + + Stitch + .collect { + spaceIdMap.values.toSeq.distinct.map { spaceId => + fetcher + .fetch(spaceId, lookupContext) + .map { + case Fetch.Result(Some(audioSpace), _) if audioSpace.broadcastId.nonEmpty => + Some(spaceId -> audioSpace) + case _ => None + } + } + }.map { results => + val audioSpaceMap = results.flatten.toMap + candidates.map { candidate => + val spaceState = spaceIdMap.get(candidate.candidate.id).flatMap { spaceId => + audioSpaceMap.get(spaceId).flatMap(_.state) + } + + FeatureMapBuilder().add(SpaceStateFeature, spaceState).build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala new file mode 100644 index 000000000..fbe86e697 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala @@ -0,0 +1,174 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.contentrecommender.{thriftscala => cr} +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature +import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.clients.strato.topics.TopicSocialProofClient +import com.twitter.topiclisting.TopicListingViewerContext +import com.twitter.tsp.thriftscala.TopicFollowType +import com.twitter.tsp.{thriftscala => tsp} +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]] +object TSPInferredTopicDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TSPInferredTopicFeatureHydrator @Inject() ( + topicSocialProofClient: TopicSocialProofClient) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic") + + override val features: Set[Feature[_, _]] = Set( + TSPInferredTopicFeature, + TSPInferredTopicDataRecordFeature, + TopicIdSocialContextFeature, + TopicContextFunctionalityTypeFeature + ) + + private val topK = 3 + + private val SuggestTypesToSetSocialProof: Set[hmt.ServedType] = Set( + hmt.ServedType.ForYouTweetMixer, + hmt.ServedType.ForYouSimclusters, + hmt.ServedType.ForYouTwhin, + hmt.ServedType.ForYouUtg, + hmt.ServedType.ForYouUvg, + hmt.ServedType.ForYouUteg, + hmt.ServedType.ForYouPopularGeo, + hmt.ServedType.ForYouPopularTopic, + hmt.ServedType.ForYouDeepRetrieval, + hmt.ServedType.ForYouEvergreenDeepRetrieval, + hmt.ServedType.ForYouRelatedCreator, + hmt.ServedType.ForYouLocal, + hmt.ServedType.ForYouTrends, + hmt.ServedType.ForYouHistoryAuthor, + ) + + private val DefaultFeatureMap = FeatureMap( + TSPInferredTopicFeature, + Map.empty[Long, Double], + TSPInferredTopicDataRecordFeature, + new DataRecord(), + TopicIdSocialContextFeature, + None, + TopicContextFunctionalityTypeFeature, + None + ) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val tags = candidates.collect { + case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn => + candidate.candidate.id -> candidate.features + .getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag]) + }.toMap + + val followableUttTopics = query.features + .flatMap(_.getOrElse(FollowableUttTopicsFeatures, None)) + .map(_.mapValues(_.getOrElse(TopicFollowType.ImplicitFollow))) + + val topicSocialProofRequest = tsp.TopicSocialProofRequest( + userId = query.getRequiredUserId, + tweetIds = candidates.map(_.candidate.id).toSet, + displayLocation = cr.DisplayLocation.HomeTimeline, + topicListingSetting = tsp.TopicListingSetting.Followable, + context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift, + bypassModes = None, + allowlist = followableUttTopics, + // Only TweetMixer source has this data. Convert the TweetMixer metric tag to tsp metric tag. + tags = if (tags.isEmpty) None else Some(tags) + ) + + topicSocialProofClient + .getTopicTweetSocialProofResponse(topicSocialProofRequest) + .map { + case Some(response) => + handleResponse(response, candidates) + case _ => candidates.map { _ => DefaultFeatureMap } + } + } + + private def handleResponse( + response: tsp.TopicSocialProofResponse, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[FeatureMap] = { + candidates.map { candidate => + val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty) + if (topicWithScores.nonEmpty) { + val (socialProofId, socialProofFunctionalityType) = + if (SuggestTypesToSetSocialProof.contains( + candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined))) { + getSocialProof(topicWithScores) + } else (None, None) + + val inferredTopicFeatures = + topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap + + val inferredTopicDataRecord = + InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head + + FeatureMap( + TSPInferredTopicFeature, + inferredTopicFeatures, + TSPInferredTopicDataRecordFeature, + inferredTopicDataRecord, + TopicIdSocialContextFeature, + socialProofId, + TopicContextFunctionalityTypeFeature, + socialProofFunctionalityType + ) + } else DefaultFeatureMap + } + } + + private def getSocialProof( + topicWithScores: Seq[tsp.TopicWithScore] + ): (Option[Long], Option[TopicContextFunctionalityType]) = { + val followingTopicId = topicWithScores.collectFirst { + case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) => topicId + } + + if (followingTopicId.nonEmpty) { + return (followingTopicId, Some(BasicTopicContextFunctionalityType)) + } + + val implicitFollowingId = topicWithScores.collectFirst { + case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) => + topicId + } + + if (implicitFollowingId.nonEmpty) { + return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType)) + } + + (None, None) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TransformerPostEmbeddingFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TransformerPostEmbeddingFeatureHydrator.scala new file mode 100644 index 000000000..9b3e210a7 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TransformerPostEmbeddingFeatureHydrator.scala @@ -0,0 +1,155 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.PostTransformerEmbeddingsHomeBlueAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.PostTransformerEmbeddingsHomeGreenAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.PostTransformerEmbeddingsJointBlueAdapter +import com.twitter.home_mixer_features.{thriftscala => hmf} +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TransformerPostEmbeddingHomeBlueFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +object TransformerPostEmbeddingHomeGreenFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +object TransformerPostEmbeddingJointBlueFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TransformerPostEmbeddingHomeBlueFeatureHydrator @Inject() ( + homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, + statsReceiver: StatsReceiver) + extends TransformerPostEmbeddingFeatureHydrator( + homeMixerFeatureService, + statsReceiver, + hmf.Cache.TransformerPostEmbeddings, + TransformerPostEmbeddingHomeBlueFeature, + PostTransformerEmbeddingsHomeBlueAdapter + ) { + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "TransformerPostEmbeddingBlue") +} + +@Singleton +class TransformerPostEmbeddingHomeGreenFeatureHydrator @Inject() ( + homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, + statsReceiver: StatsReceiver) + extends TransformerPostEmbeddingFeatureHydrator( + homeMixerFeatureService, + statsReceiver, + hmf.Cache.TransformerPostEmbeddingsGreen, + TransformerPostEmbeddingHomeGreenFeature, + PostTransformerEmbeddingsHomeGreenAdapter + ) { + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "TransformerPostEmbeddingGreen") +} + +@Singleton +class TransformerPostEmbeddingJointBlueFeatureHydrator @Inject() ( + homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, + statsReceiver: StatsReceiver) + extends TransformerPostEmbeddingFeatureHydrator( + homeMixerFeatureService, + statsReceiver, + hmf.Cache.TransformerPostJointEmbeddingsBlue, + TransformerPostEmbeddingJointBlueFeature, + PostTransformerEmbeddingsJointBlueAdapter + ) { + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "TransformerPostEmbeddingGreen") +} + +abstract class TransformerPostEmbeddingFeatureHydrator( + homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, + statsReceiver: StatsReceiver, + cache: hmf.Cache, + feature: DataRecordInAFeature[TweetCandidate], + dataRecordAdapter: TimelinesMutatingAdapterBase[Option[ml.FloatTensor]]) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val features: Set[Feature[_, _]] = Set(feature) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyTotalCounter = scopedStatsReceiver.counter("key/total") + + private val batchSize = 50 + + private def getEmbeddingsFromHMF( + tweetIds: Seq[Long] + ): Future[Seq[Option[Seq[Double]]]] = { + val keysSerialized = tweetIds.map(_.toString) + val request = hmf.HomeMixerFeaturesRequest(keysSerialized, cache) + val responseFut = + homeMixerFeatureService.getHomeMixerFeatures(request) + responseFut + .map { response => + response.homeMixerFeatures + .map { homeMixerFeatureOpt => + homeMixerFeatureOpt.homeMixerFeaturesType.map { + case hmf.HomeMixerFeaturesType.RawEmbedding(rawEmbedding) => + rawEmbedding + case _ => throw new Exception("Unknown type returned") + } + } + }.handle { case _ => Seq.fill(tweetIds.size)(None) } + } + + private def getBatchedFeatureMap( + candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]] + ): Future[Seq[FeatureMap]] = { + val tweetIds = + candidatesBatch.map { candidate => + keyTotalCounter.incr() + candidate.candidate.id + } + + getEmbeddingsFromHMF(tweetIds).map { response => + response.map { embeddingOpt => + val floatTensor = + embeddingOpt.map { embedding => + keyFoundCounter.incr() + ml.FloatTensor(embedding) + } + val dataRecord = + dataRecordAdapter.adaptToDataRecords(floatTensor).asScala.head + FeatureMap(feature, dataRecord) + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + OffloadFuturePools.offloadBatchSeqToFutureSeq(candidates, getBatchedFeatureMap, batchSize) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetEntityServiceContentFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetEntityServiceContentFeatureHydrator.scala new file mode 100644 index 000000000..ab2035d3c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetEntityServiceContentFeatureHydrator.scala @@ -0,0 +1,254 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.knuddels.jtokkit.Encodings +import com.knuddels.jtokkit.api.Encoding +import com.knuddels.jtokkit.api.ModelType +import com.twitter.escherbird.{thriftscala => esb} +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.ContentFeatureAdapter +import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature +import com.twitter.home_mixer.model.HomeFeatures.HasMultipleMedia +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature +import com.twitter.home_mixer.model.HomeFeatures.IsSelfThreadFeature +import com.twitter.home_mixer.model.HomeFeatures.MediaCategoryFeature +import com.twitter.home_mixer.model.HomeFeatures.MediaIdFeature +import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.SemanticAnnotationIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFromTweetypieFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetTextTokensFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoAspectRatioFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoHeightFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoWidthFeature +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetypieContentFeaturesDeciderParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetypieContentFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetypieContentMediaEntityFeaturesParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentFeatureFromTesService +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer.util.ObservedKeyValueResultHandler +import com.twitter.home_mixer.util.tweetypie.content.FeatureExtractionHelper +import com.twitter.home_mixer_features.{thriftscala => hmf} +import com.twitter.mediaservices.commons.thriftscala.MediaCategory +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.common.util.MediaUnderstandingAnnotations +import com.twitter.tweetypie.{thriftscala => tp} +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TweetypieContentDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +case class TweetContentExtractionResult( + annotations: Seq[esb.TweetEntityAnnotation] = Seq.empty, + contentDataRecord: DataRecord = new DataRecord(), + videoDurationMs: Option[Int] = None, + tweetLanguage: Option[String] = None, + tweetText: Option[String] = None, + tweetTextTokens: Option[Seq[Int]] = None, + aspectRatio: Option[Float] = None, + height: Option[Short] = None, + width: Option[Short] = None, + isSelfThread: Boolean = false, + mediaId: Option[Long] = None, + mediaCategory: Option[MediaCategory] = None, + hasMultipleMedia: Option[Boolean] = None, + hasImage: Option[Boolean] = None, + hasVideo: Option[Boolean] = None) + +@Singleton +class TweetEntityServiceContentFeatureHydrator @Inject() ( + homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, + override val statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with ObservedKeyValueResultHandler + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "TweetEntityServiceContent") + + override val features: Set[Feature[_, _]] = Set( + MediaUnderstandingAnnotationIdsFeature, + SemanticAnnotationIdsFeature, + TweetypieContentDataRecordFeature, + VideoDurationMsFeature, + TweetLanguageFromTweetypieFeature, + TweetTextFeature, + TweetTextTokensFeature, + VideoAspectRatioFeature, + VideoHeightFeature, + VideoWidthFeature, + IsSelfThreadFeature, + MediaIdFeature, + MediaCategoryFeature, + HasMultipleMedia, + HasImageFeature, + HasVideoFeature + ) + + override def onlyIf( + query: PipelineQuery + ): Boolean = query.params(EnableTweetypieContentFeaturesDeciderParam) && + query.params(EnableTweetypieContentFeaturesParam) + + override val statScope: String = identifier.toString + private val batchSize = 64 + val tokenizer: Encoding = + Encodings.newLazyEncodingRegistry().getEncodingForModel(ModelType.GPT_4) + + private def getContentFeaturesFromHMF( + tweetIdsToHydrate: Seq[Long], + getFromTES: Boolean = false + ): Future[Seq[Option[tp.Tweet]]] = { + val keysSerialized = tweetIdsToHydrate.map(_.toString) + val request = hmf.HomeMixerFeaturesRequest( + keysSerialized, + hmf.Cache.TweetypieContent, + Some( + hmf.HomeMixerFeaturesRequestContext.ContentFeatureRequestContext( + hmf.ContentFeatureRequestContext(Some(getFromTES)) + )) + ) + val responseFut = + homeMixerFeatureService.getHomeMixerFeatures(request) + responseFut + .map { response => + response.homeMixerFeatures + .map { homeMixerFeatureOpt => + homeMixerFeatureOpt.homeMixerFeaturesType.map { + case hmf.HomeMixerFeaturesType.TweetypieContent(homeMixerFeature) => + homeMixerFeature + case _ => throw new Exception("Unknown type returned") + } + } + }.handle { case _ => Seq.fill(tweetIdsToHydrate.size)(None) } + } + + private def getFeatureMaps( + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + query: PipelineQuery, + getFromTes: Boolean + ): Future[Seq[FeatureMap]] = { + val tweetIdsToHydrate = candidates.map(CandidatesUtil.getOriginalTweetId) + + val isExtractMediaEntities = query.params(EnableTweetypieContentMediaEntityFeaturesParam) + + val responseMap = getContentFeaturesFromHMF(tweetIdsToHydrate, getFromTes) + + responseMap.map { result => + result.map { tweetContent => + val transformed = postTransformer(tweetContent, isExtractMediaEntities) + val annotationIds = transformed.annotations.map(_.entityId) + val mediaUnderstandingAnnotationIds = + getNonSensitiveHighRecallMediaUnderstandingAnnotationEntityIds(transformed.annotations) + FeatureMapBuilder(sizeHint = 13) + .add(MediaUnderstandingAnnotationIdsFeature, mediaUnderstandingAnnotationIds) + .add(SemanticAnnotationIdsFeature, annotationIds) + .add(TweetypieContentDataRecordFeature, transformed.contentDataRecord) + .add(VideoDurationMsFeature, transformed.videoDurationMs) + .add(TweetLanguageFromTweetypieFeature, transformed.tweetLanguage) + .add(TweetTextFeature, transformed.tweetText) + .add(TweetTextTokensFeature, transformed.tweetTextTokens) + .add(VideoAspectRatioFeature, transformed.aspectRatio) + .add(VideoHeightFeature, transformed.height) + .add(VideoWidthFeature, transformed.width) + .add(IsSelfThreadFeature, transformed.isSelfThread) + .add(MediaIdFeature, transformed.mediaId) + .add(MediaCategoryFeature, transformed.mediaCategory) + .add(HasMultipleMedia, transformed.hasMultipleMedia.getOrElse(false)) + .add(HasImageFeature, transformed.hasImage.getOrElse(false)) + .add(HasVideoFeature, transformed.hasVideo.getOrElse(false)) + .build() + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val getFromTes = query.params(EnableContentFeatureFromTesService) + OffloadFuturePools.offloadBatchSeqToFutureSeq( + candidates, + getFeatureMaps(_, query, getFromTes), + batchSize) + } + + private def postTransformer( + result: Option[tp.Tweet], + isExtractMediaEntities: Boolean = true + ): TweetContentExtractionResult = { + val transformedValue = + result.map(FeatureExtractionHelper.extractFeatures(_, isExtractMediaEntities)) + val semanticAnnotations = + transformedValue.flatMap { _.semanticCoreAnnotations }.getOrElse(Seq.empty) + val dataRecord = ContentFeatureAdapter.adaptToDataRecords(transformedValue).asScala.head + val videoDurationMs = transformedValue.flatMap { _.videoDurationMs } + + val mediaId = transformedValue.flatMap { _.media.flatMap(_.headOption).map(_.mediaId) } + val hasMultipleMedia = + Some(transformedValue.map(_.media.map(_.size > 1).getOrElse(false)).getOrElse(false)) + val mediaCategory = transformedValue.flatMap { + _.media.flatMap(_.headOption).flatMap(_.mediaKey).map(_.mediaCategory) + } + val tweetLanguage = result.flatMap { _.language.map(_.language) } + val tweetText = result.flatMap { _.coreData.map(_.text) } + val tweetTextTokens = tweetText.map { text => + tokenizer.encodeOrdinary(text, 1024).getTokens.toArray.toSeq + } + val aspectRatioNum = transformedValue.flatMap { _.aspectRatioNum } + val aspectRatioDen = transformedValue.flatMap { _.aspectRatioDen } + val aspectRatio = aspectRatioNum + .zip(aspectRatioDen).map { + case (num, den) => + if (den != 0) num.toFloat / den.toFloat + else -1 + }.find(_ > 0) + val mediaHeight = transformedValue.flatMap { _.heights.flatMap(_.headOption) } + val mediaWidth = transformedValue.flatMap { _.widths.flatMap(_.headOption) } + val isSelfThread = transformedValue.exists(_.selfThreadMetadata.nonEmpty) + val hasImage = transformedValue.flatMap(_.hasImage) + val hasVideo = transformedValue.flatMap(_.hasVideo) + TweetContentExtractionResult( + semanticAnnotations, + dataRecord, + videoDurationMs, + tweetLanguage, + tweetText, + tweetTextTokens, + aspectRatio, + mediaHeight, + mediaWidth, + isSelfThread, + mediaId, + mediaCategory, + hasMultipleMedia, + hasImage, + hasVideo + ) + } + + private def getNonSensitiveHighRecallMediaUnderstandingAnnotationEntityIds( + semanticCoreAnnotations: Seq[esb.TweetEntityAnnotation] + ): Seq[Long] = semanticCoreAnnotations + .filter(MediaUnderstandingAnnotations.isEligibleNonSensitiveHighRecallMUAnnotation) + .map(_.entityId) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetEntityServiceFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetEntityServiceFeatureHydrator.scala new file mode 100644 index 000000000..b5aa0f4ce --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetEntityServiceFeatureHydrator.scala @@ -0,0 +1,376 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.conversions.DurationOps.RichDuration +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.FirstMediaIdFeature +import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsArticleFeature +import com.twitter.home_mixer.model.HomeFeatures.IsInReplyToReplyOrDirectedFeature +import com.twitter.home_mixer.model.HomeFeatures.IsInReplyToRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature +import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.OriginalTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TesBatchedStratoClient +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache +import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor +import com.twitter.mediaservices.commons.thriftscala.MediaKey +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.TtlCache +import com.twitter.servo.keyvalue.KeyValueResult +import com.twitter.stitch.Arrow +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Client +import com.twitter.strato.client.CommunityId +import com.twitter.strato.client.UserId +import com.twitter.strato.generated.client.tweetypie.federated.ArticleOnTweetClientColumn +import com.twitter.strato.generated.client.tweetypie.federated.CommunityOnTweetClientColumn +import com.twitter.strato.generated.client.tweetypie.federated.ContextualQuoteTweetRefOnTweetClientColumn +import com.twitter.strato.generated.client.tweetypie.federated.DirectedAtUserOnTweetClientColumn +import com.twitter.strato.generated.client.tweetypie.federated.ExclusiveTweetControlOnTweetClientColumn +import com.twitter.strato.generated.client.tweetypie.federated.MediaKeysOnTweetClientColumn +import com.twitter.strato.generated.client.tweetypie.federated.MentionsOnTweetClientColumn +import com.twitter.strato.generated.client.tweetypie.federated.NarrowcastPlaceOnTweetClientColumn +import com.twitter.strato.generated.client.tweetypie.federated.PureCoreDataOnTweetClientColumn +import com.twitter.strato.generated.client.tweetypie.federated.UrlsOnTweetClientColumn +import com.twitter.strato.graphql.contextual_refs.thriftscala.ContextualTweetRef +import com.twitter.tweetypie.{thriftscala => tp} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class TweetEntityServiceFeatureHydrator @Inject() ( + @Named(TweetypieStaticEntitiesCache) cacheClient: TtlCache[Long, tp.Tweet], + @Named(TesBatchedStratoClient) stratoClient: Client, + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + import TweetEntityServiceFeatureHydrator._ + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(IdentifierName) + + override val features: Set[Feature[_, _]] = Features + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val pureCoreDataNonEmptyCounter = scopedStatsReceiver.counter("pureCoreDataNonEmpty") + private val pureCoreDataEmptyCounter = scopedStatsReceiver.counter("pureCoreDataEmpty") + + private lazy val getTESRecord: Arrow[Long, TESRecord] = getTESRecordArrow(stratoClient) + + private lazy val getFromTES: Arrow[Seq[Long], Map[Long, tp.Tweet]] = Arrow + .sequence(getTESRecord) + .map { record => + record + .map(_.getTweetOpt).filter { tweetOpt => + if (tweetOpt.nonEmpty) pureCoreDataNonEmptyCounter.incr() + else pureCoreDataEmptyCounter.incr() + tweetOpt.nonEmpty + }.map(tweet => (tweet.get.id, tweet.get)).toMap + } + + private lazy val getHydratedTweetMapWithCacheWriteBack: Arrow[ + KeyValueResult[Long, tp.Tweet], + Map[Long, tp.Tweet] + ] = Arrow + .zipWithArg( + Arrow + .identity[KeyValueResult[Long, tp.Tweet]] + .andThen(getFromTES.contramap[KeyValueResult[Long, tp.Tweet]](kv => kv.notFound.toSeq)) + ).map { + case (fromCache, fromTES) => + fromTES.map(kv => cacheClient.set(kv._1, kv._2, CacheTTL)) + fromCache.found ++ fromTES + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val tweetIds: Seq[Long] = candidates.map(_.candidate.id) + val inReplyToIds = candidates.flatMap(_.features.getOrElse(InReplyToTweetIdFeature, None)) + val idsToHydrate = (tweetIds ++ inReplyToIds).distinct + + Stitch + .callFuture(cacheClient.get(idsToHydrate)) + .flatMap(cacheResult => getHydratedTweetMapWithCacheWriteBack(cacheResult)) + .map { tweetMap: Map[Long, tp.Tweet] => + candidates.map { candidate => + tweetMap + .get(candidate.candidate.id).map { tweet => + getFeatureMapFromTweet( + tweet, + candidate.features.getOrElse(InReplyToTweetIdFeature, None).flatMap(tweetMap.get) + ) + }.getOrElse(DefaultFeatureMap) + } + } + } +} + +object TweetEntityServiceFeatureHydrator { + private val IdentifierName = "TweetEntityService" + private val CacheTTL = 48.hours + + private val Features: Set[Feature[_, _]] = Set( + AuthorIdFeature, + CommunityIdFeature, + DirectedAtUserIdFeature, + ExclusiveConversationAuthorIdFeature, + FirstMediaIdFeature, + HasImageFeature, + HasVideoFeature, + IsArticleFeature, + InReplyToTweetIdFeature, + InReplyToUserIdFeature, + IsInReplyToReplyOrDirectedFeature, + IsInReplyToRetweetFeature, + IsRetweetFeature, + LocationIdFeature, + MentionScreenNameFeature, + MentionUserIdFeature, + QuotedTweetIdFeature, + OriginalTweetIdFeature, + SourceTweetIdFeature, + SourceUserIdFeature, + TweetMediaIdsFeature, + TweetUrlsFeature, + ) + + private val DefaultFeatureMap = FeatureMapBuilder() + .add(AuthorIdFeature, None) + .add(DirectedAtUserIdFeature, None) + .add(ExclusiveConversationAuthorIdFeature, None) + .add(HasImageFeature, false) + .add(HasVideoFeature, false) + .add(IsArticleFeature, false) + .add(InReplyToTweetIdFeature, None) + .add(InReplyToUserIdFeature, None) + .add(IsInReplyToReplyOrDirectedFeature, false) + .add(IsInReplyToRetweetFeature, false) + .add(IsRetweetFeature, false) + .add(LocationIdFeature, None) + .add(MentionScreenNameFeature, Seq.empty) + .add(MentionUserIdFeature, Seq.empty) + .add(QuotedTweetIdFeature, None) + .add(OriginalTweetIdFeature, None) + .add(SourceTweetIdFeature, None) + .add(SourceUserIdFeature, None) + .add(CommunityIdFeature, None) + .add(TweetMediaIdsFeature, Seq.empty) + .add(FirstMediaIdFeature, None) + .add(TweetUrlsFeature, Seq.empty) + .build() + + private def getFeatureMapFromTweet( + tweet: tp.Tweet, + inReplyToTweet: Option[tp.Tweet] + ): FeatureMap = { + val coreData = tweet.coreData + val quotedTweet = tweet.quotedTweet + val mentions = tweet.mentions.getOrElse(Seq.empty) + val share = coreData.flatMap(_.share) + val reply = coreData.flatMap(_.reply) + val urls = tweet.urls.map(_.flatMap(_.expanded)).toSeq.flatten + + val (isInReplyToReplyOrDirected, isInReplyToRetweet) = inReplyToTweet + .map { tweet => + ( + // when inReplyToUserId exists, it can be a reply or a directedAt tweet, + // depending on whether inReplyToTweetId exists + tweet.coreData.flatMap(_.reply).map(_.inReplyToUserId).isDefined, + tweet.coreData.flatMap(_.share).isDefined + ) + }.getOrElse((false, false)) + + // There are cases where the inReplyToUserId exists while inReplyToStatusId does not. + // They're usually directed tweets that are not replies. + val inReplyToTweetId = reply.flatMap(_.inReplyToStatusId) + val inReplyToUserId = if (inReplyToTweetId.nonEmpty) reply.map(_.inReplyToUserId) else None + val tweetMediaIds = TweetMediaFeaturesExtractor.getMediaIds(tweet) + + FeatureMapBuilder() + .add(AuthorIdFeature, coreData.map(_.userId)) + .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) + .add( + ExclusiveConversationAuthorIdFeature, + tweet.exclusiveTweetControl.map(_.conversationAuthorId)) + .add(HasImageFeature, TweetMediaFeaturesExtractor.hasImage(tweet)) + .add(HasVideoFeature, TweetMediaFeaturesExtractor.hasVideo(tweet)) + .add(IsArticleFeature, tweet.article.isDefined) + .add(InReplyToTweetIdFeature, inReplyToTweetId) + .add(InReplyToUserIdFeature, inReplyToUserId) + .add(IsRetweetFeature, share.isDefined) + .add(IsInReplyToReplyOrDirectedFeature, isInReplyToReplyOrDirected) + .add(IsInReplyToRetweetFeature, isInReplyToRetweet) + .add(LocationIdFeature, tweet.narrowcastPlace.map(_.id)) + .add(MentionScreenNameFeature, mentions.map(_.screenName)) + .add(MentionUserIdFeature, mentions.flatMap(_.userId)) + .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) + .add(OriginalTweetIdFeature, Some(share.map(_.sourceStatusId).getOrElse(tweet.id))) + .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) + .add(SourceUserIdFeature, share.map(_.sourceUserId)) + .add(CommunityIdFeature, tweet.communities.flatMap(_.communityIds.headOption)) + .add(TweetMediaIdsFeature, tweetMediaIds) + .add(FirstMediaIdFeature, tweetMediaIds.headOption) + .add(TweetUrlsFeature, urls) + .build() + } + + private def getTESRecordArrow(stratoClient: Client): Arrow[Long, TESRecord] = { + val pureCoreDataArrow: Arrow[Long, Option[tp.PureCoreData]] = + new PureCoreDataOnTweetClientColumn(stratoClient).fetcher.asArrow + .contramap[Long](tweetId => (tweetId, ())) + .map(_.v) + + val communityArrow: Arrow[Long, Option[CommunityId]] = + new CommunityOnTweetClientColumn(stratoClient).fetcher.asArrow + .contramap[Long](tweetId => (tweetId, ())) + .map(_.v) + + val directedAtUserArrow: Arrow[Long, Option[UserId]] = + new DirectedAtUserOnTweetClientColumn(stratoClient).fetcher.asArrow + .contramap[Long](tweetId => (tweetId, ())) + .map(_.v) + + val exclusiveTweetControlArrow: Arrow[Long, Option[tp.ExclusiveTweetControl]] = + new ExclusiveTweetControlOnTweetClientColumn(stratoClient).fetcher.asArrow + .contramap[Long](tweetId => (tweetId, ())) + .map(_.v) + + val mediaKeysArrow: Arrow[Long, Option[Seq[MediaKey]]] = + new MediaKeysOnTweetClientColumn(stratoClient).fetcher.asArrow + .contramap[Long](tweetId => (tweetId, ())) + .map(_.v) + + val mentionsArrow: Arrow[Long, Option[Seq[tp.MentionEntity]]] = + new MentionsOnTweetClientColumn(stratoClient).fetcher.asArrow + .contramap[Long](tweetId => (tweetId, ())) + .map(_.v) + + val contextualQuoteTweetRefArrow: Arrow[Long, Option[ContextualTweetRef]] = + new ContextualQuoteTweetRefOnTweetClientColumn(stratoClient).fetcher.asArrow + .contramap[Long](tweetId => (tweetId, ())) + .map(_.v) + + val narrowcastPlaceArrow: Arrow[Long, Option[tp.NarrowcastPlace]] = + new NarrowcastPlaceOnTweetClientColumn(stratoClient).fetcher.asArrow + .contramap[Long](tweetId => (tweetId, ())) + .map(_.v) + + val articleArrow: Arrow[Long, Option[tp.Article]] = + new ArticleOnTweetClientColumn(stratoClient).fetcher.asArrow + .contramap[Long](tweetId => (tweetId, ())) + .map(_.v) + + val urlsArrow: Arrow[Long, Option[Seq[tp.UrlEntity]]] = + new UrlsOnTweetClientColumn(stratoClient).fetcher.asArrow + .contramap[Long](tweetId => (tweetId, ())) + .map(_.v) + + Arrow + .zipWithArg( + Arrow + .identity[Long].andThen(Arrow.join( + pureCoreDataArrow, + articleArrow, + communityArrow, + directedAtUserArrow, + exclusiveTweetControlArrow, + mediaKeysArrow, + mentionsArrow, + narrowcastPlaceArrow, + contextualQuoteTweetRefArrow, + urlsArrow + )) + ).map { + case ( + tweetId, + ( + pureCoreDataOpt: Option[tp.PureCoreData], + articleOpt: Option[tp.Article], + communityIdOpt: Option[CommunityId], + directedAtUserIdOpt: Option[UserId], + exclusiveTweetControls: Option[tp.ExclusiveTweetControl], + mediaKeysOpt: Option[Seq[MediaKey]], + mentionEntitiesOpt: Option[Seq[tp.MentionEntity]], + narrowcastPlaceOpt: Option[tp.NarrowcastPlace], + quotedTweetOpt: Option[ContextualTweetRef], + urlsOpt: Option[Seq[tp.UrlEntity]] + )) => + TESRecord( + tweetId, + pureCoreDataOpt, + articleOpt, + communityIdOpt, + directedAtUserIdOpt, + exclusiveTweetControls, + mediaKeysOpt, + mentionEntitiesOpt, + narrowcastPlaceOpt, + quotedTweetOpt, + urlsOpt + ) + } + } +} + +case class TESRecord( + tweetId: Long, + pureCoreDataOpt: Option[tp.PureCoreData], + articleOpt: Option[tp.Article], + communityIdOpt: Option[CommunityId], + directedAtUserIdOpt: Option[UserId], + exclusiveTweetControls: Option[tp.ExclusiveTweetControl], + mediaKeysOpt: Option[Seq[MediaKey]], + mentionEntitiesOpt: Option[Seq[tp.MentionEntity]], + narrowcastPlaceOpt: Option[tp.NarrowcastPlace], + quotedTweetOpt: Option[ContextualTweetRef], + urlsOpt: Option[Seq[tp.UrlEntity]]) { + + def getTweetOpt: Option[tp.Tweet] = pureCoreDataOpt.map { pureCoreData => + tp.Tweet( + id = tweetId, + coreData = Some( + tp.TweetCoreData.unsafeEmpty.copy( + userId = pureCoreData.userId, + share = pureCoreData.share, + reply = pureCoreData.reply, + directedAtUser = directedAtUserIdOpt.map(id => tp.DirectedAtUser(id.value, "")) + )), + article = articleOpt, + communities = communityIdOpt.map(community => tp.Communities(Seq(community.value))), + exclusiveTweetControl = exclusiveTweetControls + .map(control => tp.ExclusiveTweetControl(control.conversationAuthorId)), + media = mediaKeysOpt.map { mediaKeys => + mediaKeys.map { key => + tp.MediaEntity.unsafeEmpty.copy(mediaId = key.mediaId, mediaKey = Some(key)) + } + }, + mentions = mentionEntitiesOpt, + narrowcastPlace = narrowcastPlaceOpt, + quotedTweet = quotedTweetOpt.map(qt => tp.QuotedTweet.unsafeEmpty.copy(tweetId = qt.id)), + urls = urlsOpt + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetLanguageFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetLanguageFeatureHydrator.scala new file mode 100644 index 000000000..3a5a68ae8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetLanguageFeatureHydrator.scala @@ -0,0 +1,61 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFromLanguageSignalFeature +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.language.tweet.LanguageOnTweetClientColumn +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.util.OffloadFuturePools +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TweetLanguageFeatureHydrator @Inject() ( + languageOnTweetClientColumn: LanguageOnTweetClientColumn, + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TweetLanguage") + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val failedCounter = scopedStatsReceiver.scope(getClass.getSimpleName).counter("failure") + + private val DefaultFeatureMap = + FeatureMapBuilder().add(TweetLanguageFromLanguageSignalFeature, None).build() + + override def features: Set[Feature[_, _]] = Set(TweetLanguageFromLanguageSignalFeature) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + OffloadFuturePools.offloadStitch { + Stitch.collect { + candidates.map { candidate => + languageOnTweetClientColumn.fetcher + .fetch( + CandidatesUtil.getOriginalTweetId(candidate), + LanguageOnTweetClientColumn + .View(true)).map { result => + FeatureMapBuilder() + .add(TweetLanguageFromLanguageSignalFeature, result.v) + .build() + }.rescue { + case _ => + failedCounter.incr() + Stitch.value(DefaultFeatureMap) + } + } + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetLargeEmbeddingsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetLargeEmbeddingsFeatureHydrator.scala new file mode 100644 index 000000000..b36645625 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetLargeEmbeddingsFeatureHydrator.scala @@ -0,0 +1,145 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.TweetLargeEmbeddingsFeature +import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.TweetLargeEmbeddingsKeyFeature +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam +import com.twitter.home_mixer_features.{thriftscala => hmf} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.adapters.large_embeddings.HashingFeatureParams +import com.twitter.timelines.prediction.adapters.large_embeddings.HomeMixerLargeEmbeddingsFeatureHydrator +import com.twitter.timelines.prediction.adapters.large_embeddings.LargeEmbeddingsAdapter +import com.twitter.timelines.prediction.adapters.large_embeddings.TweetLargeEmbeddingsAdapter +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TweetLargeEmbeddingsFeatureHydrator @Inject() ( + statsReceiver: StatsReceiver, + override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] + with HomeMixerLargeEmbeddingsFeatureHydrator { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TweetLargeEmbeddings") + + override val features: Set[Feature[_, _]] = + Set(TweetLargeEmbeddingsFeature, TweetLargeEmbeddingsKeyFeature) + + override val adapter: LargeEmbeddingsAdapter = TweetLargeEmbeddingsAdapter + + override val cacheType: hmf.Cache = hmf.Cache.TweetLargeEmbeddings + + override val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableLargeEmbeddingsFeatureHydrationParam) + + override val modelName2HashingFeatureParams: Map[String, HashingFeatureParams] = Map( + "hr_video_prod__v3_realtime" -> HashingFeatureParams( + scales = Seq(1487022661L, 1399245971L), + biases = Seq(1372992088L, 632996194L), + modulus = 2865175829L, + bucketSize = 10000000L, + ), + "hr_video_prod__v2_lembeds" -> HashingFeatureParams( + scales = Seq(2516541900L, 2376187492L), + biases = Seq(3022238687L, 1571354734L), + modulus = 3047336911L, + bucketSize = 1000000L, + ), + "hr_prod__v4_embeds_230M" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + "hr_prod__v5_embeds_230M_and_transformer" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + "hr_prod__v5_watchtime" -> HashingFeatureParams( + scales = Seq(407033648L, 940305868L), + biases = Seq(494266171L, 269596788L), + modulus = 949146421L, + bucketSize = 100000000L, + ), + "hr_prod__v6_transformer_v2" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + "hr_prod__v6_mixed_training" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + "hr_prod__v6_transformer_v2_kafka_merge_join" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + "hr_prod__v6_transformer_v2_realtime_debias_21apr" -> HashingFeatureParams( + scales = Seq(2161410491L, 1754358832L), + biases = Seq(296686044L, 1959990826L), + modulus = 2361375383L, + bucketSize = 100000000L, + ), + ) + + // Hashing Features + override val defaultHashingFeatureParams: HashingFeatureParams = HashingFeatureParams( + scales = Seq(1131302000L, 303023026L), + biases = Seq(799473858L, 600426834L), + modulus = 3588720353L, + bucketSize = 10000000L, + ) + + private val batchSize = 25 + + private def getBatchedFeatureMap( + modelName: String, + candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], + ): Future[Seq[FeatureMap]] = { + val tweetIds = candidatesBatch.map { candidate => candidate.candidate.id } + + getLargeEmbeddings(tweetIds, modelName).map { responses => + responses.map { largeEmbeddingResponse => + FeatureMapBuilder() + .add(TweetLargeEmbeddingsFeature, largeEmbeddingResponse.dataRecord) + .add(TweetLargeEmbeddingsKeyFeature, largeEmbeddingResponse.hashedKeys) + .build() + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val modelName = query.params(ModelNameParam) + OffloadFuturePools.offloadBatchSeqToFutureSeq( + candidates, + getBatchedFeatureMap(modelName, _), + batchSize + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetMetaDataFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetMetaDataFeatureHydrator.scala new file mode 100644 index 000000000..c9c89180d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetMetaDataFeatureHydrator.scala @@ -0,0 +1,61 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.RichDataRecord +import com.twitter.ml.api.constant.SharedFeatures +import com.twitter.ml.api.util.DataRecordConverters._ +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import java.lang.{Long => JLong} + +object TweetMetaDataDataRecord + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +object TweetMetaDataFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetMetaData") + + override def features: Set[Feature[_, _]] = Set(TweetMetaDataDataRecord) + + private val batchSize = 64 + + def getFeatureMap(candidate: CandidateWithFeatures[TweetCandidate]): FeatureMap = { + val richDataRecord = new RichDataRecord() + setFeatures(richDataRecord, candidate.candidate, candidate.features) + FeatureMap(TweetMetaDataDataRecord, richDataRecord.getRecord) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + OffloadFuturePools.offloadBatchElementToElement(candidates, getFeatureMap, batchSize) + } + + private def setFeatures( + richDataRecord: RichDataRecord, + candidate: TweetCandidate, + existingFeatures: FeatureMap + ): Unit = { + richDataRecord.setFeatureValue[JLong](SharedFeatures.TWEET_ID, candidate.id) + + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, + CandidatesUtil.getOriginalAuthorId(existingFeatures)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetTimeFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetTimeFeatureHydrator.scala new file mode 100644 index 000000000..f367afb53 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetTimeFeatureHydrator.scala @@ -0,0 +1,129 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature +import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetAgeFeature +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.util.FDsl._ +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._ +import com.twitter.util.Duration + +import scala.collection.Searching._ + +object TweetTimeDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +object TweetTimeFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with WithDefaultFeatureMap { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetTime") + + override val features: Set[Feature[_, _]] = Set(TweetTimeDataRecordFeature, TweetAgeFeature) + + override val defaultFeatureMap: FeatureMap = + FeatureMap( + TweetTimeDataRecordFeature, + TweetTimeDataRecordFeature.defaultValue, + TweetAgeFeature, + None + ) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { + + val nonPollingTimestampsMs = query.features.get.getOrElse(NonPollingTimesFeature, Seq.empty) + + candidates.map { candidate => + val tweetFeatures = candidate.features.getOrElse(EarlybirdFeature, None) + val timeSinceTweetCreation = + SnowflakeId.timeFromIdOpt(candidate.candidate.id).map(query.queryTime.since) + val timeSinceTweetCreationMs = timeSinceTweetCreation.map(_.inMillis) + + val timeSinceSourceTweetCreationOpt = candidate.features + .getOrElse(SourceTweetIdFeature, None) + .flatMap { sourceTweetId => + SnowflakeId.timeFromIdOpt(sourceTweetId).map(query.queryTime.since) + }.orElse(timeSinceTweetCreation) + + val lastFavSinceCreationHrs = + tweetFeatures.flatMap(_.lastFavSinceCreationHrs).map(_.toDouble) + val lastRetweetSinceCreationHrs = + tweetFeatures.flatMap(_.lastRetweetSinceCreationHrs).map(_.toDouble) + val lastReplySinceCreationHrs = + tweetFeatures.flatMap(_.lastReplySinceCreationHrs).map(_.toDouble) + val lastQuoteSinceCreationHrs = + tweetFeatures.flatMap(_.lastQuoteSinceCreationHrs).map(_.toDouble) + val timeSinceLastFavoriteHrs = + getTimeSinceLastEngagementHrs(lastFavSinceCreationHrs, timeSinceSourceTweetCreationOpt) + val timeSinceLastRetweetHrs = + getTimeSinceLastEngagementHrs(lastRetweetSinceCreationHrs, timeSinceSourceTweetCreationOpt) + val timeSinceLastReplyHrs = + getTimeSinceLastEngagementHrs(lastReplySinceCreationHrs, timeSinceSourceTweetCreationOpt) + val timeSinceLastQuoteHrs = + getTimeSinceLastEngagementHrs(lastQuoteSinceCreationHrs, timeSinceSourceTweetCreationOpt) + + val timeSinceLastNonPollingRequest = + nonPollingTimestampsMs.headOption.map(query.queryTime.inMillis - _) + + val nonPollingRequestsSinceTweetCreation = + if (nonPollingTimestampsMs.nonEmpty && timeSinceTweetCreationMs.isDefined) { + nonPollingTimestampsMs + .search(timeSinceTweetCreationMs.get)(Ordering[Long].reverse) + .insertionPoint + } else 0.0 + + val tweetAgeRatio = + if (timeSinceTweetCreationMs.exists(_ > 0.0) && timeSinceLastNonPollingRequest.isDefined) { + timeSinceLastNonPollingRequest.get / timeSinceTweetCreationMs.get.toDouble + } else 0.0 + + val dataRecord = new DataRecord() + .setFeatureValue(IS_TWEET_RECYCLED, false) + .setFeatureValue(TWEET_AGE_RATIO, tweetAgeRatio) + .setFeatureValueFromOption( + TIME_SINCE_TWEET_CREATION, + timeSinceTweetCreationMs.map(_.toDouble) + ) + .setFeatureValue( + NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, + nonPollingRequestsSinceTweetCreation + ) + .setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, lastFavSinceCreationHrs) + .setFeatureValueFromOption(LAST_RETWEET_SINCE_CREATION_HRS, lastRetweetSinceCreationHrs) + .setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, lastReplySinceCreationHrs) + .setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, lastQuoteSinceCreationHrs) + .setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, timeSinceLastFavoriteHrs) + .setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, timeSinceLastRetweetHrs) + .setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, timeSinceLastReplyHrs) + .setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, timeSinceLastQuoteHrs) + + FeatureMap(TweetTimeDataRecordFeature, dataRecord, TweetAgeFeature, timeSinceTweetCreationMs) + } + } + + private def getTimeSinceLastEngagementHrs( + lastEngagementTimeSinceCreationHrsOpt: Option[Double], + timeSinceTweetCreation: Option[Duration] + ): Option[Double] = lastEngagementTimeSinceCreationHrsOpt.flatMap { lastEngagementTimeHrs => + timeSinceTweetCreation.map(_.inHours - lastEngagementTimeHrs) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetTypeMetricsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetTypeMetricsFeatureHydrator.scala new file mode 100644 index 000000000..19d8e4a38 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetTypeMetricsFeatureHydrator.scala @@ -0,0 +1,50 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates +import com.twitter.home_mixer.model.HomeFeatures.TweetTypeMetricsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.suggests.controller_data.Home + +object TweetTypeMetricsFeatureHydrator + extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetTypeMetrics") + + override val features: Set[Feature[_, _]] = Set(TweetTypeMetricsFeature) + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + existingFeatures: FeatureMap + ): Stitch[FeatureMap] = { + // Tweet type metrics are already available for cached tweets and shouldn't be overwritten + val tweetTypeMetricsFeature = existingFeatures.getOrElse(TweetTypeMetricsFeature, None) + val queryFeatures = query.features.getOrElse(FeatureMap.empty) + + val tweetTypesByteList = if (tweetTypeMetricsFeature.isEmpty) { + val bitset = new java.util.BitSet() + + val trueTweetTypes = HomeTweetTypePredicates.PredicateMap.collect { + // Not combining query and candidate features to reduce cost, instead running predicate separately + case (predicateName, predicate) + if (predicate(existingFeatures) | predicate(queryFeatures)) => + predicateName + }.toSet + + Home.TweetTypeIdxMap.collect { + case (tweetType, index) if trueTweetTypes.contains(tweetType) => bitset.set(index) + } + + Some(bitset.toByteArray.toList) + } else tweetTypeMetricsFeature + + Stitch.value(FeatureMapBuilder().add(TweetTypeMetricsFeature, tweetTypesByteList).build()) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala index f2effa1b2..4e1994189 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala @@ -5,8 +5,8 @@ import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsArticleFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature @@ -17,10 +17,12 @@ import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.util.tweetypie.RequestFields +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetEntityServiceMigrationParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate @@ -33,15 +35,19 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.spam.rtf.{thriftscala => rtf} import com.twitter.stitch.Stitch import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} +import com.twitter.strato.client.Client +import com.twitter.strato.generated.client.tweetypie.managed.HomeMixerOnTweetClientColumn import com.twitter.tweetypie.{thriftscala => tp} import com.twitter.util.logging.Logging import javax.inject.Inject +import javax.inject.Named import javax.inject.Singleton @Singleton class TweetypieFeatureHydrator @Inject() ( tweetypieStitchClient: TweetypieStitchClient, - statsReceiver: StatsReceiver) + statsReceiver: StatsReceiver, + @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client) extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Logging { @@ -49,12 +55,14 @@ class TweetypieFeatureHydrator @Inject() ( override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, + CommunityIdFeature, ExclusiveConversationAuthorIdFeature, InReplyToTweetIdFeature, + IsArticleFeature, IsHydratedFeature, IsNsfw, - IsNsfwFeature, IsRetweetFeature, + LocationIdFeature, QuotedTweetDroppedFeature, QuotedTweetIdFeature, QuotedUserIdFeature, @@ -65,45 +73,49 @@ class TweetypieFeatureHydrator @Inject() ( VisibilityReason ) + val HydrationFields: Set[tp.TweetInclude] = Set( + tp.TweetInclude.TweetFieldId(tp.Tweet.CommunitiesField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.CoreDataField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.SelfThreadMetadataField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.ExclusiveTweetControlField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.IdField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.LanguageField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.QuotedTweetField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.ArticleField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.NarrowcastPlaceField.id) + ) + + private val tweetypieTweetsFoundCounter = + statsReceiver.counter("TweetypieTweetsFound") + private val tweetypieTweetsNotFoundCounter = + statsReceiver.counter("TweetypieTweetsNotFound") + private val tesTweetsFoundCounter = + statsReceiver.counter("TesTweetsFound") + private val tesTweetsNotFoundCounter = + statsReceiver.counter("TesTweetsNotFound") + private val DefaultFeatureMap = FeatureMapBuilder() + .add(CommunityIdFeature, None) + .add(IsArticleFeature, false) .add(IsHydratedFeature, false) .add(IsNsfw, None) - .add(IsNsfwFeature, false) + .add(LocationIdFeature, None) .add(QuotedTweetDroppedFeature, false) .add(TweetTextFeature, None) .add(VisibilityReason, None) .build() - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, + private def buildFeatureMap( + gtfResult: Stitch[tp.GetTweetFieldsResult], + fromTes: Boolean, + exclusiveAuthorIdOpt: Option[Long], existingFeatures: FeatureMap ): Stitch[FeatureMap] = { - val safetyLevel = query.product match { - case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest - case ForYouProduct => - val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true) - if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations - case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome - case ListTweetsProduct => rtf.SafetyLevel.TimelineLists - case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed - case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown") - } - - val tweetFieldsOptions = tp.GetTweetFieldsOptions( - tweetIncludes = RequestFields.TweetTPHydrationFields, - includeRetweetedTweet = true, - includeQuotedTweet = true, - visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, - safetyLevel = Some(safetyLevel), - forUserId = query.getOptionalUserId - ) - - val exclusiveAuthorIdOpt = - existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None) - - tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map { + gtfResult.map { case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) => + if (fromTes) tesTweetsFoundCounter.incr() + else tweetypieTweetsFoundCounter.incr() + val coreData = found.tweet.coreData val isNsfwAdmin = coreData.exists(_.nsfwAdmin) val isNsfwUser = coreData.exists(_.nsfwUser) @@ -143,14 +155,21 @@ class TweetypieFeatureHydrator @Inject() ( val isNsfw = isNsfwAdmin || isNsfwUser || sourceTweetIsNsfw || quotedTweetIsNsfw + val tpExclusiveAuthorIdOpt = found.tweet.exclusiveTweetControl.map(_.conversationAuthorId) + val updatedExclusiveAuthorId = tpExclusiveAuthorIdOpt.orElse(exclusiveAuthorIdOpt) + + val communityId = found.tweet.communities.flatMap(_.communityIds.headOption) + FeatureMapBuilder() .add(AuthorIdFeature, tweetAuthorId) - .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt) + .add(CommunityIdFeature, communityId) + .add(ExclusiveConversationAuthorIdFeature, updatedExclusiveAuthorId) .add(InReplyToTweetIdFeature, inReplyToTweetId) + .add(IsArticleFeature, found.tweet.article.nonEmpty) .add(IsHydratedFeature, true) .add(IsNsfw, Some(isNsfw)) - .add(IsNsfwFeature, isNsfw) .add(IsRetweetFeature, retweetedTweetId.isDefined) + .add(LocationIdFeature, found.tweet.narrowcastPlace.map(_.id)) .add(QuotedTweetDroppedFeature, quotedTweetDropped) .add(QuotedTweetIdFeature, quotedTweetId) .add(QuotedUserIdFeature, quotedTweetUserId) @@ -161,13 +180,16 @@ class TweetypieFeatureHydrator @Inject() ( .add(VisibilityReason, found.suppressReason) .build() - // If no tweet result found, return default and pre-existing features case _ => + if (fromTes) tesTweetsNotFoundCounter.incr() + else tweetypieTweetsNotFoundCounter.incr() + DefaultFeatureMap ++ FeatureMapBuilder() .add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt) .add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) .add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) + .add(LocationIdFeature, existingFeatures.getOrElse(LocationIdFeature, None)) .add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) .add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) .add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) @@ -176,4 +198,51 @@ class TweetypieFeatureHydrator @Inject() ( .build() } } + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + existingFeatures: FeatureMap + ): Stitch[FeatureMap] = { + val safetyLevel = query.product match { + case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest + case ForYouProduct => + val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true) + if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations + case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome + case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed + case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown") + } + + val tweetFieldsOptions = tp.GetTweetFieldsOptions( + tweetIncludes = HydrationFields, + includeRetweetedTweet = true, + includeQuotedTweet = true, + visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, + safetyLevel = Some(safetyLevel), + forUserId = query.getOptionalUserId + ) + + val exclusiveAuthorIdOpt = + existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None) + + if (query.params(EnableTweetEntityServiceMigrationParam)) { + val fetcher = new HomeMixerOnTweetClientColumn(stratoClient).fetcher + fetcher + .fetch( + candidate.id, + tweetFieldsOptions + ).map(_.v).flatMap { + case Some(result) => + buildFeatureMap(Stitch.value(result), true, exclusiveAuthorIdOpt, existingFeatures) + case None => + tesTweetsNotFoundCounter.incr() + Stitch.value(DefaultFeatureMap) + } + } else { + val gtfResult: Stitch[tp.GetTweetFieldsResult] = + tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions) + buildFeatureMap(gtfResult, fromTes = false, exclusiveAuthorIdOpt, existingFeatures) + } + } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala new file mode 100644 index 000000000..6cc78ec11 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala @@ -0,0 +1,86 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinAuthorFollowEmbeddingsAdapter +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureRepository +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer.util.ObservedKeyValueResultHandler +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.repository.KeyValueRepository +import com.twitter.servo.repository.KeyValueResult +import com.twitter.stitch.Stitch +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TwhinAuthorFollowFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TwhinAuthorFollowFeatureHydrator @Inject() ( + @Named(TwhinAuthorFollowFeatureRepository) + client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], + override val statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with ObservedKeyValueResultHandler { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TwhinAuthorFollow") + + override val features: Set[Feature[_, _]] = Set(TwhinAuthorFollowFeature) + + override val statScope: String = identifier.toString + + private val emptyDataRecord = new DataRecord() + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val possiblyAuthorIds = extractKeys(candidates) + val authorIds = possiblyAuthorIds.flatten.distinct + + val response: Future[KeyValueResult[Long, DataRecord]] = + if (authorIds.isEmpty) Future.value(KeyValueResult.empty) + else client(authorIds).map(_.mapFound(postTransformer)) + + response.map { result => + possiblyAuthorIds.map { possiblyAuthorId => + val value = + observedGet(key = possiblyAuthorId, keyValueResult = result) + .map(_.getOrElse(emptyDataRecord)) + + FeatureMapBuilder().add(TwhinAuthorFollowFeature, value).build() + } + } + } + + private def postTransformer(embedding: ml.FloatTensor): DataRecord = { + TwhinAuthorFollowEmbeddingsAdapter.adaptToDataRecords(Some(embedding)).asScala.head + } + + private def extractKeys( + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[Option[Long]] = { + candidates.map { candidate => + CandidatesUtil.getOriginalAuthorId(candidate.features) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildTweetFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildTweetFeatureHydrator.scala new file mode 100644 index 000000000..af76a31b5 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildTweetFeatureHydrator.scala @@ -0,0 +1,127 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinRebuildTweetEmbeddingsAdapter +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinRebuildTweetEmbeddingsStore +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableHomeMixerFeaturesService +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer_features.{thriftscala => hmf} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.util.Future +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.simclusters_v2.thriftscala.TwhinEmbeddingDataset +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TwhinRebuildTweetFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TwhinRebuildTweetFeatureHydrator @Inject() ( + homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, + @Named(TwhinRebuildTweetEmbeddingsStore) store: ReadableStore[(Long, Long), TwhinTweetEmbedding], + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "TwhinRebuildTweet") + + override val features: Set[Feature[_, _]] = Set(TwhinRebuildTweetFeature) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyTotalCounter = scopedStatsReceiver.counter("key/total") + + private val batchSize = 50 + + private val versionId = TwhinEmbeddingDataset.RefreshedTwhinTweet.value.toLong + + private def getTwhinEmbeddingsFromHMF( + originalTweetIds: Seq[Long] + ): Future[Seq[Option[TwhinTweetEmbedding]]] = { + val keysSerialized = originalTweetIds.map(_.toString) + val request = hmf.HomeMixerFeaturesRequest(keysSerialized, hmf.Cache.TwhinRebuild) + val responseFut = + homeMixerFeatureService.getHomeMixerFeatures(request) + responseFut + .map { response => + response.homeMixerFeatures + .map { homeMixerFeatureOpt => + homeMixerFeatureOpt.homeMixerFeaturesType.map { + case hmf.HomeMixerFeaturesType.TwhinTweetEmbedding(homeMixerFeature) => + homeMixerFeature + case _ => throw new Exception("Unknown type returned") + } + } + }.handle { case _ => Seq.fill(originalTweetIds.size)(None) } + } + + private def getTwhinEmbeddingsFromReadableStore( + originalTweetIds: Seq[Long] + ): Future[Seq[Option[TwhinTweetEmbedding]]] = { + val tweetIdVersionIdPairs = originalTweetIds.map(tweetId => (tweetId, versionId)) + Future.collect(store.multiGet(tweetIdVersionIdPairs.toSet)).map { storeResponse => + tweetIdVersionIdPairs.map { + storeResponse.getOrElse(_, None) + } + } + } + + private def getBatchedFeatureMap( + candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], + callMiddleMan: Boolean + ): Future[Seq[FeatureMap]] = { + val originalTweetIds = + candidatesBatch.map { candidate => + { + keyTotalCounter.incr() + CandidatesUtil.getOriginalTweetId(candidate.candidate, candidate.features) + } + } + + val responseMap = + if (callMiddleMan) getTwhinEmbeddingsFromHMF(originalTweetIds) + else getTwhinEmbeddingsFromReadableStore(originalTweetIds) + + responseMap.map { response => + response.map { twhinEmbeddingOpt => + val floatTensor = { + keyFoundCounter.incr() + twhinEmbeddingOpt.map(twhinEmbedding => ml.FloatTensor(twhinEmbedding.embedding)) + } + val dataRecord = + TwhinRebuildTweetEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head + FeatureMap(TwhinRebuildTweetFeature, dataRecord) + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val callMiddleMan = query.params(EnableHomeMixerFeaturesService) + OffloadFuturePools.offloadBatchSeqToFutureSeq( + candidates, + getBatchedFeatureMap(_, callMiddleMan), + batchSize) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildUserEngagementQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildUserEngagementQueryFeatureHydrator.scala new file mode 100644 index 000000000..016dd4262 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildUserEngagementQueryFeatureHydrator.scala @@ -0,0 +1,74 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinRebuildUserEngagementEmbeddingsAdapter +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinRebuildUserEngagementFeatureRepository +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.repository.KeyValueRepository +import com.twitter.stitch.Stitch +import com.twitter.util.Return +import com.twitter.util.Throw +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TwhinRebuildUserEngagementFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TwhinRebuildUserEngagementQueryFeatureHydrator @Inject() ( + @Named(TwhinRebuildUserEngagementFeatureRepository) + client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TwhinRebuildUserEngagement") + + override val features: Set[Feature[_, _]] = Set(TwhinRebuildUserEngagementFeature) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") + private val keyTotalCounter = scopedStatsReceiver.counter("key/total") + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + Stitch.callFuture(client(Seq(userId))).map { results => + keyTotalCounter.incr() + val embedding: Option[ml.FloatTensor] = results(userId) match { + case Return(value) => + if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() + else keyNotFoundCounter.incr() + value + case Throw(_) => + keyFailureCounter.incr() + None + case _ => + None + } + + val dataRecord = + TwhinRebuildUserEngagementEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head + + FeatureMapBuilder() + .add(TwhinRebuildUserEngagementFeature, dataRecord) + .build() + } + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildUserPositiveQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildUserPositiveQueryFeatureHydrator.scala new file mode 100644 index 000000000..552d0cf5f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildUserPositiveQueryFeatureHydrator.scala @@ -0,0 +1,67 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinRebuildUserPositiveEmbeddingsAdapter +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinRebuildUserPositiveEmbeddingsStore +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.simclusters_v2.thriftscala.TwhinEmbeddingDataset +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TwhinRebuildUserPositiveFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TwhinRebuildUserPositiveQueryFeatureHydrator @Inject() ( + @Named(TwhinRebuildUserPositiveEmbeddingsStore) store: ReadableStore[ + (Long, Long), + TwhinTweetEmbedding + ], + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TwhinRebuildUserPositive") + + override val features: Set[Feature[_, _]] = Set(TwhinRebuildUserPositiveFeature) + + private val versionId = TwhinEmbeddingDataset.RefreshedTwhinTweet.value + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + private val keyTotalCounter = scopedStatsReceiver.counter("key/total") + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + Stitch + .callFuture(store.get((query.getRequiredUserId, versionId))).map { resultOpt => + keyTotalCounter.incr() + resultOpt match { + case Some(_) => + keyFoundCounter.incr() + case None => keyNotFoundCounter.incr() + } + val floatTensor = resultOpt.map(result => ml.FloatTensor(result.embedding)) + val dataRecord = TwhinRebuildUserPositiveEmbeddingsAdapter + .adaptToDataRecords(floatTensor).asScala.head + FeatureMapBuilder().add(TwhinRebuildUserPositiveFeature, dataRecord).build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinTweetFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinTweetFeatureHydrator.scala new file mode 100644 index 000000000..90205eb10 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinTweetFeatureHydrator.scala @@ -0,0 +1,111 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinTweetEmbeddingsAdapter +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinTweetEmbeddingsStore +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableHomeMixerFeaturesService +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer_features.{thriftscala => hmf} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.util.Future +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TwhinTweetFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TwhinTweetFeatureHydrator @Inject() ( + homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, + @Named(TwhinTweetEmbeddingsStore) store: ReadableStore[Long, TwhinTweetEmbedding]) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinTweet") + + override val features: Set[Feature[_, _]] = Set(TwhinTweetFeature) + + private val batchSize = 50 + + private def getTwhinEmbeddingsFromHMF( + originalTweetIds: Seq[Long] + ): Future[Seq[Option[TwhinTweetEmbedding]]] = { + val keysSerialized = originalTweetIds.map(_.toString) + val request = hmf.HomeMixerFeaturesRequest(keysSerialized, hmf.Cache.Twhin) + val responseFut = + homeMixerFeatureService.getHomeMixerFeatures(request) + responseFut + .map { response => + response.homeMixerFeatures + .map { homeMixerFeatureOpt => + homeMixerFeatureOpt.homeMixerFeaturesType.map { + case hmf.HomeMixerFeaturesType.TwhinTweetEmbedding(homeMixerFeature) => + homeMixerFeature + case _ => throw new Exception("Unknown type returned") + } + } + }.handle { case _ => Seq.fill(originalTweetIds.size)(None) } + } + + private def getTwhinEmbeddingsFromReadableStore( + originalTweetIds: Seq[Long] + ): Future[Seq[Option[TwhinTweetEmbedding]]] = { + Future.collect(store.multiGet(originalTweetIds.toSet)).map { storeResponse => + originalTweetIds.map { + storeResponse.getOrElse(_, None) + } + } + } + + private def getBatchedFeatureMap( + candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], + callMiddleMan: Boolean + ): Future[Seq[FeatureMap]] = { + val originalTweetIds = + candidatesBatch.map { candidate => + CandidatesUtil.getOriginalTweetId(candidate.candidate, candidate.features) + } + + val responseMap = + if (callMiddleMan) getTwhinEmbeddingsFromHMF(originalTweetIds) + else getTwhinEmbeddingsFromReadableStore(originalTweetIds) + + responseMap.map { response => + response.map { twhinEmbeddingOpt => + val floatTensor = + twhinEmbeddingOpt.map(twhinEmbedding => ml.FloatTensor(twhinEmbedding.embedding)) + val dataRecord = + TwhinTweetEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head + FeatureMap(TwhinTweetFeature, dataRecord) + } + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val callMiddleMan = query.params(EnableHomeMixerFeaturesService) + OffloadFuturePools.offloadBatchSeqToFutureSeq( + candidates, + getBatchedFeatureMap(_, callMiddleMan), + batchSize) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala new file mode 100644 index 000000000..ee30ba504 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala @@ -0,0 +1,72 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserEngagementEmbeddingsAdapter +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserEngagementFeatureRepository +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.repository.KeyValueRepository +import com.twitter.stitch.Stitch +import com.twitter.util.Return +import com.twitter.util.Throw +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TwhinUserEngagementFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TwhinUserEngagementQueryFeatureHydrator @Inject() ( + @Named(TwhinUserEngagementFeatureRepository) + client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TwhinUserEngagement") + + override val features: Set[Feature[_, _]] = Set(TwhinUserEngagementFeature) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + Stitch.callFuture(client(Seq(userId))).map { results => + val embedding: Option[ml.FloatTensor] = results(userId) match { + case Return(value) => + if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() + else keyNotFoundCounter.incr() + value + case Throw(_) => + keyFailureCounter.incr() + None + case _ => + None + } + + val dataRecord = + TwhinUserEngagementEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head + + FeatureMapBuilder() + .add(TwhinUserEngagementFeature, dataRecord) + .build() + } + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala new file mode 100644 index 000000000..afb836807 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala @@ -0,0 +1,72 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserFollowEmbeddingsAdapter +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserFollowFeatureRepository +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.repository.KeyValueRepository +import com.twitter.stitch.Stitch +import com.twitter.util.Return +import com.twitter.util.Throw +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TwhinUserFollowFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TwhinUserFollowQueryFeatureHydrator @Inject() ( + @Named(TwhinUserFollowFeatureRepository) + client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TwhinUserFollow") + + override val features: Set[Feature[_, _]] = Set(TwhinUserFollowFeature) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + Stitch + .callFuture(client(Seq(userId))) + .map { results => + val embedding: Option[ml.FloatTensor] = results(userId) match { + case Return(value) => + if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() + else keyNotFoundCounter.incr() + value + case Throw(_) => + keyFailureCounter.incr() + None + case _ => + None + } + + val dataRecord = TwhinUserFollowEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head + + FeatureMapBuilder() + .add(TwhinUserFollowFeature, dataRecord) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserNegativeFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserNegativeFeatureHydrator.scala new file mode 100644 index 000000000..c6b7b6f71 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserNegativeFeatureHydrator.scala @@ -0,0 +1,47 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserNegativeEmbeddingsAdapter +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserNegativeEmbeddingsStore +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TwhinUserNegativeFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TwhinUserNegativeQueryFeatureHydrator @Inject() ( + @Named(TwhinUserNegativeEmbeddingsStore) store: ReadableStore[Long, TwhinTweetEmbedding]) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TwhinUserNegative") + + override val features: Set[Feature[_, _]] = Set(TwhinUserNegativeFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + Stitch.callFuture(store.get(query.getRequiredUserId)).map { resultOpt => + val floatTensor = resultOpt.map(result => ml.FloatTensor(result.embedding)) + val dataRecord = TwhinUserNegativeEmbeddingsAdapter + .adaptToDataRecords(floatTensor).asScala.head + FeatureMapBuilder().add(TwhinUserNegativeFeature, dataRecord).build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserPositiveFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserPositiveFeatureHydrator.scala new file mode 100644 index 000000000..2f838ed37 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserPositiveFeatureHydrator.scala @@ -0,0 +1,47 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserPositiveEmbeddingsAdapter +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserPositiveEmbeddingsStore +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TwhinUserPositiveFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TwhinUserPositiveQueryFeatureHydrator @Inject() ( + @Named(TwhinUserPositiveEmbeddingsStore) store: ReadableStore[Long, TwhinTweetEmbedding]) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TwhinUserPositive") + + override val features: Set[Feature[_, _]] = Set(TwhinUserPositiveFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + Stitch.callFuture(store.get(query.getRequiredUserId)).map { resultOpt => + val floatTensor = resultOpt.map(result => ml.FloatTensor(result.embedding)) + val dataRecord = TwhinUserPositiveEmbeddingsAdapter + .adaptToDataRecords(floatTensor).asScala.head + FeatureMapBuilder().add(TwhinUserPositiveFeature, dataRecord).build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinVideoFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinVideoFeatureHydrator.scala new file mode 100644 index 000000000..3d3c53a84 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinVideoFeatureHydrator.scala @@ -0,0 +1,54 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinVideoEmbeddingsAdapter +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinVideoEmbeddingsStore +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TwhinVideoFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TwhinVideoFeatureHydrator @Inject() ( + @Named(TwhinVideoEmbeddingsStore) store: ReadableStore[Long, TwhinTweetEmbedding]) + extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinVideo") + + override val features: Set[Feature[_, _]] = Set(TwhinVideoFeature) + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + existingFeatures: FeatureMap + ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { + val originalTweetId = CandidatesUtil.getOriginalTweetId(candidate, existingFeatures) + + store.get(originalTweetId).map { resultOpt => + val floatTensor = resultOpt.map(result => ml.FloatTensor(result.embedding)) + val dataRecord = TwhinVideoEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head + FeatureMapBuilder().add(TwhinVideoFeature, dataRecord).build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UnifiedUserActionsUserIdentifierFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UnifiedUserActionsUserIdentifierFeatureHydrator.scala new file mode 100644 index 000000000..ec2f8a451 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UnifiedUserActionsUserIdentifierFeatureHydrator.scala @@ -0,0 +1,60 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.unified_counter.service.UuaUserIdentifierClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UnifiedUserActionsUserIdentifierFeatureHydrator @Inject() ( + uuaUserIdentifierClientColumn: UuaUserIdentifierClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UnifiedUserActionsUserIdentifier") + + override val features: Set[Feature[_, _]] = Set( + UuaUserGenderFeature, + UuaUserStateFeature, + UuaUserAgeBucketFeature + ) + + private val DefaultFeatureMap = FeatureMapBuilder() + .add(UuaUserGenderFeature, None) + .add(UuaUserStateFeature, None) + .add(UuaUserAgeBucketFeature, None) + .build() + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + uuaUserIdentifierClientColumn.fetcher + .fetch(query.getRequiredUserId.toString) + .map { response => + response.v match { + case Some(userInfo) => + val gender = userInfo.userGender.map(_.toString) + val state = userInfo.userState.map(_.value.toLong) + val ageBucket = userInfo.userAgeBucket.map(_.toString) + + FeatureMapBuilder() + .add(UuaUserGenderFeature, gender) + .add(UuaUserStateFeature, state) + .add(UuaUserAgeBucketFeature, ageBucket) + .build() + + case _ => + DefaultFeatureMap + } + } + .rescue { + case _: Throwable => + Stitch.value(DefaultFeatureMap) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserActionByteArrayQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserActionByteArrayQueryFeatureHydrator.scala new file mode 100644 index 000000000..d3d66ea33 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserActionByteArrayQueryFeatureHydrator.scala @@ -0,0 +1,99 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.UserActionsByteArrayFeature +import com.twitter.home_mixer.param.HomeGlobalParams.UserActionsMaxCount +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.user_history_transformer.user_actions.UserActionSequenceMhClientColumn +import com.twitter.user_history_transformer.domain.AggregationAlgorithmV1 +import com.twitter.user_history_transformer.domain.AggregationConfig +import com.twitter.user_history_transformer.domain.AggregationProcessor +import com.twitter.user_history_transformer.domain.UserActionSequenceUtils +import com.twitter.user_history_transformer.util.SchemaUtils +import com.x.user_action_sequence.thriftscala.UserActionSequenceDataContainer.OrderedAggregatedUserActionList +import com.x.user_action_sequence.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserActionsArrayByteQueryFeatureHydrator @Inject() ( + userActionSequenceMhClientColumn: UserActionSequenceMhClientColumn, + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserActionsByteArray") + + override val features: Set[Feature[_, _]] = Set(UserActionsByteArrayFeature) + + private val DefaultFeatureMap = FeatureMap(UserActionsByteArrayFeature, None) + + private val windowTimeMs = 5 * 60 * 1000 + + private val aggregationProcessor = new AggregationProcessor( + AggregationConfig( + postProcessorSeq = Seq.empty, + windowTimeMs = windowTimeMs, + maxLength = 1024, + aggregationAlgorithm = AggregationAlgorithmV1 + ) + ) + + private def hasNegativeValue(value: Option[Long]): Boolean = value.exists(_ < 0) + + private def hasNegativeValues(aggregatedUserAction: t.AggregatedUserAction): Boolean = { + if (hasNegativeValue(aggregatedUserAction.userId)) return true + + aggregatedUserAction.tweetInfo.exists { tweetInfo => + val fieldsToCheck = List( + tweetInfo.tweetId, + tweetInfo.authorId, + tweetInfo.retweetingTweetId, + tweetInfo.quotingTweetId, + tweetInfo.replyingTweetId, + tweetInfo.quotedTweetId, + tweetInfo.inReplyToTweetId, + tweetInfo.retweetedTweetId, + tweetInfo.editedTweetId + ) + + fieldsToCheck.exists(hasNegativeValue) + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = + OffloadFuturePools.offloadStitch { + userActionSequenceMhClientColumn.fetcher.fetch(query.getRequiredUserId).map { response => + val featureMap = response.v.map { userActionsSeq => + val decompressedUserActionSeq = + UserActionSequenceUtils.expand(userActionsSeq, statsReceiver) + val downsampledUserActionSeq = + UserActionSequenceUtils.mhToKafka(decompressedUserActionSeq) + val aggregatedUserActions = aggregationProcessor + .process(downsampledUserActionSeq) + .filterNot(hasNegativeValues) + .takeRight(query.params(UserActionsMaxCount)) + + val filteredUserActionSeq = userActionsSeq.copy( + userActionsData = Some( + OrderedAggregatedUserActionList( + t.AggregatedUserActionList(aggregatedUserActions = Some(aggregatedUserActions)) + ) + ) + ) + + val actions = SchemaUtils.convertUserActionSequenceThriftToProtobuf(filteredUserActionSeq) + + FeatureMap(UserActionsByteArrayFeature, Some(actions.toByteArray)) + } + + featureMap.getOrElse(DefaultFeatureMap) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserActionsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserActionsQueryFeatureHydrator.scala new file mode 100644 index 000000000..f30b9581e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserActionsQueryFeatureHydrator.scala @@ -0,0 +1,156 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.UserActionsContainsExplicitSignalsFeature +import com.twitter.home_mixer.model.HomeFeatures.UserActionsFeature +import com.twitter.home_mixer.model.HomeFeatures.UserActionsSizeFeature +import com.twitter.home_mixer.param.HomeGlobalParams.UserActionsMaxCount +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableDenseUserActionsHydrationParam +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.user_history_transformer.user_actions.UserActionSequenceMhClientColumn +import com.twitter.user_history_transformer.domain.AggregationAlgorithmV1 +import com.twitter.user_history_transformer.domain.AggregationAlgorithmWithoutHomeFilter +import com.twitter.user_history_transformer.domain.AggregationConfig +import com.twitter.user_history_transformer.domain.AggregationProcessor +import com.twitter.user_history_transformer.domain.UserActionSequenceUtils +import com.twitter.user_history_transformer.util.SchemaUtils +import com.x.user_action_sequence.thriftscala.ActionName.ClientTweetRecapDwelled +import com.x.user_action_sequence.thriftscala.ActionName.ClientTweetRecapNotDwelled +import com.x.user_action_sequence.thriftscala.ActionName +import com.x.user_action_sequence.thriftscala.AggregatedUserAction +import com.x.user_action_sequence.thriftscala.UserActionSequenceDataContainer.OrderedAggregatedUserActionList +import com.x.user_action_sequence.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserActionsQueryFeatureHydrator @Inject() ( + userActionSequenceMhClientColumn: UserActionSequenceMhClientColumn, + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserActions") + + override val features: Set[Feature[_, _]] = Set( + UserActionsFeature, + UserActionsSizeFeature, + UserActionsContainsExplicitSignalsFeature + ) + + private val userAggregatedActionSeqlengthStat = + statsReceiver.stat("UserActionsQueryFeatureHydrator", "length") + + private val DefaultFeatureMap = FeatureMapBuilder() + .add(UserActionsFeature, None) + .add(UserActionsSizeFeature, None) + .add(UserActionsContainsExplicitSignalsFeature, false) + .build() + + private val windowTimeMs = 5 * 60 * 1000 + + private val ExcludedDwellActions: Set[ActionName] = + Set(ClientTweetRecapDwelled, ClientTweetRecapNotDwelled) + + private def filterDwells( + aggAction: AggregatedUserAction + ): Boolean = { + aggAction.actions + .getOrElse(Seq.empty) + .exists { + _.actionName.exists { name => + !ExcludedDwellActions.contains(name) + } + } + } + + private val aggregationProcessor = new AggregationProcessor( + AggregationConfig( + postProcessorSeq = Seq.empty, + windowTimeMs = windowTimeMs, + maxLength = 1024, + aggregationAlgorithm = AggregationAlgorithmV1, + ) + ) + + private val denseAggregationProcessor = new AggregationProcessor( + AggregationConfig( + postProcessorSeq = Seq.empty, + windowTimeMs = windowTimeMs, + maxLength = 1024, + aggregationAlgorithm = AggregationAlgorithmWithoutHomeFilter, + aggActionFilterFuncGenerator = (_, _) => { + aggAction => filterDwells(aggAction) + } + ) + ) + + private def hasNegativeValue(value: Option[Long]): Boolean = value.exists(_ < 0) + private def hasNegativeValues(aggregatedUserAction: t.AggregatedUserAction): Boolean = { + if (hasNegativeValue(aggregatedUserAction.userId)) return true + + aggregatedUserAction.tweetInfo.exists { tweetInfo => + val fieldsToCheck = List( + tweetInfo.tweetId, + tweetInfo.authorId, + tweetInfo.retweetingTweetId, + tweetInfo.quotingTweetId, + tweetInfo.replyingTweetId, + tweetInfo.quotedTweetId, + tweetInfo.inReplyToTweetId, + tweetInfo.retweetedTweetId, + tweetInfo.editedTweetId + ) + + fieldsToCheck.exists(hasNegativeValue) + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val processor = + if (query.params(EnableDenseUserActionsHydrationParam)) denseAggregationProcessor + else aggregationProcessor + OffloadFuturePools.offloadStitch { + userActionSequenceMhClientColumn.fetcher.fetch(query.getRequiredUserId).map { response => + val featureMap = response.v.map { userActionsSeq => + val decompressedUserActionSeq = + UserActionSequenceUtils.expand(userActionsSeq, statsReceiver) + val aggregatedUserActions = processor + .process(decompressedUserActionSeq) + .filterNot(hasNegativeValues) + .takeRight(query.params(UserActionsMaxCount)) + + val size = aggregatedUserActions.length + userAggregatedActionSeqlengthStat.add(size) + + val hasExplicitSignals = + UserActionSequenceUtils.hasExplicitSignals(decompressedUserActionSeq) + + val filteredUserActionSeq = userActionsSeq.copy( + userActionsData = Some( + OrderedAggregatedUserActionList( + t.AggregatedUserActionList(aggregatedUserActions = Some(aggregatedUserActions)) + ) + ) + ) + + val actions = SchemaUtils.convertUserActionSequenceThriftToProtobuf(filteredUserActionSeq) + + FeatureMapBuilder() + .add(UserActionsFeature, Some(actions)) + .add(UserActionsSizeFeature, Some(size)) + .add(UserActionsContainsExplicitSignalsFeature, hasExplicitSignals) + .build() + } + + featureMap.getOrElse(DefaultFeatureMap) + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagedGrokCategoriesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagedGrokCategoriesFeatureHydrator.scala new file mode 100644 index 000000000..527eeea26 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagedGrokCategoriesFeatureHydrator.scala @@ -0,0 +1,45 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.trends.trip.EngagedGrokTopicsAndTagsMonthlyOnUserClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +object UserSubLevelCategoriesFeature extends Feature[TweetCandidate, Seq[(Long, Double)]] + +@Singleton +class UserEngagedGrokCategoriesFeatureHydrator @Inject() ( + engagedGrokTopicsAndTagMonthlysOnUserClientColumn: EngagedGrokTopicsAndTagsMonthlyOnUserClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserEngagedGrokCategories") + + override val features: Set[Feature[_, _]] = + Set(UserSubLevelCategoriesFeature) + + private def fetchSubLevelEntities(userId: Long): Stitch[Seq[(Long, Double)]] = { + engagedGrokTopicsAndTagMonthlysOnUserClientColumn.fetcher.fetch(userId).map { result => + result.v + .flatMap(_.subLevelEntities).getOrElse(Seq.empty) + .take(3) + .map(entityInfo => (entityInfo.entityId, entityInfo.score)) + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + fetchSubLevelEntities(userId).map { subEntities => + FeatureMapBuilder() + .add(UserSubLevelCategoriesFeature, subEntities) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagedLanguagesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagedLanguagesFeatureHydrator.scala new file mode 100644 index 000000000..213c73679 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagedLanguagesFeatureHydrator.scala @@ -0,0 +1,34 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.UserEngagedLanguagesFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.language.user.NormalizedEngagedTweetLanguagesOnUserClientColumn +import com.twitter.language.types.{thriftscala => lg} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserEngagedLanguagesFeatureHydrator @Inject() ( + engagedLanguageOnUserColumn: NormalizedEngagedTweetLanguagesOnUserClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserEngagedLanguages") + + override def features: Set[Feature[_, _]] = Set(UserEngagedLanguagesFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + engagedLanguageOnUserColumn.fetcher + .fetch(query.getRequiredUserId, Some(lg.LanguageType.User)).map { result => + FeatureMapBuilder() + .add(UserEngagedLanguagesFeature, result.v.getOrElse(Seq.empty).toSet) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagementGrokTagFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagementGrokTagFeatureHydrator.scala new file mode 100644 index 000000000..e96a0b719 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagementGrokTagFeatureHydrator.scala @@ -0,0 +1,57 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.content_understanding.UserTopTagsMhClientColumn +import com.twitter.strato.generated.client.trends.trip.UserAssociatedTopicsClientColumn +import com.twitter.trends.trip_v1.user_topics.{thriftscala => ut} + +import javax.inject.Inject +import javax.inject.Singleton + +object RealtimeUserEngagedGrokTagFeature + extends Feature[TweetCandidate, Seq[(String, Option[Double])]] +object EvergreenUserEngagedGrokTagFeature + extends Feature[TweetCandidate, Seq[(String, Option[Double])]] + +@Singleton +class UserEngagementGrokTagFeatureHydrator @Inject() ( + evergreenUserTopTags: UserTopTagsMhClientColumn, + tripUserTopTags: UserAssociatedTopicsClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "UserEngagementGrokTag") + + override val features: Set[Feature[_, _]] = + Set(RealtimeUserEngagedGrokTagFeature, EvergreenUserEngagedGrokTagFeature) + + def fetchRealTimeUserTags(userId: Long): Stitch[Seq[(String, Option[Double])]] = { + tripUserTopTags.fetcher.fetch(ut.UserTopicDomain(userId = userId, sourceId = None)).map { result => + result.v.flatMap(_.tags).getOrElse(Seq.empty[ut.TagCandidate]).map(tc => (tc.tag, tc.score)) + } + } + + def fetchEvergreenUserTags(userId: Long): Stitch[Seq[(String, Option[Double])]] = { + evergreenUserTopTags.fetcher.fetch(userId).map { view => + view.v.map(_.tags.map(tc => (tc.tag, Some(tc.score)))).getOrElse(Seq.empty) + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + Stitch.join(fetchRealTimeUserTags(userId), fetchEvergreenUserTags(userId)).map { + case (realtime, evergreen) => + FeatureMapBuilder() + .add(RealtimeUserEngagedGrokTagFeature, realtime) + .add(EvergreenUserEngagedGrokTagFeature, evergreen) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFrequentLocationHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFrequentLocationHydrator.scala new file mode 100644 index 000000000..26ea59e8c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFrequentLocationHydrator.scala @@ -0,0 +1,145 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.geoduck.common.thriftscala.TransactionLocation +import com.twitter.geoduck.common.{thriftscala => t} +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.product_mixer.component_library.feature.location.{ + Location => ProductMixerLocation +} +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ExpiringLruInProcessCache +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.geo.service.UserLocationClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +object UserFrequentLocationHydrator { + private val BaseTTLMinutes = 60 * 24 + private val TTL = (BaseTTLMinutes + scala.util.Random.nextInt(60)).minutes + + val cache: InProcessCache[Long, Option[TransactionLocation]] = + new ExpiringLruInProcessCache[Long, Option[TransactionLocation]]( + ttl = TTL, + maximumSize = 150 * 1000 // Cache up to 150k users + ) +} + +@Singleton +class UserFrequentLocationHydrator @Inject() ( + userLocationClientColumn: UserLocationClientColumn) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "UserFrequentLocationHydrator") + + override val features: Set[Feature[_, _]] = Set(LocationFeature) + + private val PlaceQuery = t.PlaceQuery( + placeTypes = Some( + Set( + t.PlaceType.Neighborhood, + t.PlaceType.City, + t.PlaceType.Metro, + t.PlaceType.Admin1, + t.PlaceType.Country + ) + ) + ) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct + + val (hitIds, missIds) = + authorIds.partition(id => UserFrequentLocationHydrator.cache.get(id).isDefined) + + val cachedOptionMap: Map[Long, Option[TransactionLocation]] = + hitIds.map(id => id -> UserFrequentLocationHydrator.cache.get(id).get).toMap + + val fetchCacheMisses: Stitch[Map[Long, TransactionLocation]] = + if (missIds.isEmpty) { + Stitch.value(Map.empty[Long, TransactionLocation]) + } else { + userLocationClientColumn.fetcher + .fetch( + key = Unit, + t.UserLocationRequest( + userIds = missIds, + placeQuery = Some(PlaceQuery) + ) + ) + .map { response => + response.v.toList.flatMap(_._1).toMap + } + .handle { + case e => + Map.empty[Long, TransactionLocation] + } + } + + val allLocationsStitch: Stitch[Map[Long, TransactionLocation]] = + fetchCacheMisses.map { fetchedMap => + missIds.foreach { id => + val locOpt: Option[TransactionLocation] = fetchedMap.get(id) + UserFrequentLocationHydrator.cache.set(id, locOpt) + } + + val cachedLocs: Map[Long, TransactionLocation] = + cachedOptionMap.collect { case (id, Some(loc)) => id -> loc } + + cachedLocs ++ fetchedMap + } + + allLocationsStitch.map { allLocations => + candidates.map { candidate => + val locOpt = for { + authorId <- candidate.features.getOrElse(AuthorIdFeature, None) + loc <- allLocations.get(authorId) + } yield loc + + locOpt + .map { transactionLocation => + val placeMap = transactionLocation.placeMap + val locationDetails = ProductMixerLocation( + neighborhood = placeMap + .flatMap(_.get(t.PlaceType.Neighborhood)) + .flatMap(_.headOption), + city = placeMap + .flatMap(_.get(t.PlaceType.City)) + .flatMap(_.headOption), + metro = placeMap + .flatMap(_.get(t.PlaceType.Metro)) + .flatMap(_.headOption), + region = placeMap + .flatMap(_.get(t.PlaceType.Admin1)) + .flatMap(_.headOption), + country = placeMap + .flatMap(_.get(t.PlaceType.Country)) + .flatMap(_.headOption) + ) + FeatureMapBuilder() + .add(LocationFeature, Option(locationDetails)) + .build() + } + .getOrElse { + FeatureMapBuilder() + .add(LocationFeature, None) + .build() + } + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFrequentLocationQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFrequentLocationQueryFeatureHydrator.scala new file mode 100644 index 000000000..c10a3c243 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFrequentLocationQueryFeatureHydrator.scala @@ -0,0 +1,90 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.product_mixer.component_library.feature.location.{Location => ProductLocation} +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.geoduck.common.{thriftscala => t} +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.geo.service.UserLocationClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserFrequentLocationQueryFeatureHydrator @Inject() ( + userLocationClientColumn: UserLocationClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserFrequentLocationQuery") + + override val features: Set[Feature[_, _]] = Set(LocationFeature) + + private val PlaceQuery = t.PlaceQuery( + placeTypes = Some( + Set( + t.PlaceType.Neighborhood, + t.PlaceType.City, + t.PlaceType.Metro, + t.PlaceType.Admin1, + t.PlaceType.Country + ) + ) + ) + + private val DefaultFeatureMap = FeatureMapBuilder() + .add(LocationFeature, None) + .build() + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + val locationStitch = userLocationClientColumn.fetcher + .fetch( + key = Unit, + t.UserLocationRequest( + userIds = Seq(userId), + placeQuery = Some(PlaceQuery) + ) + ) + .map { response => + response.v.toList.flatMap(_._1).toMap.get(userId) + } + locationStitch + .map { locationOpt => + locationOpt + .map { location => + val placeMap = location.placeMap + val locationDetails = ProductLocation( + neighborhood = placeMap + .flatMap(_.get(t.PlaceType.Neighborhood)) + .flatMap(_.headOption), + city = placeMap + .flatMap(_.get(t.PlaceType.City)) + .flatMap(_.headOption), + metro = placeMap + .flatMap(_.get(t.PlaceType.Metro)) + .flatMap(_.headOption), + region = placeMap + .flatMap(_.get(t.PlaceType.Admin1)) + .flatMap(_.headOption), + country = placeMap + .flatMap(_.get(t.PlaceType.Country)) + .flatMap(_.headOption) + ) + FeatureMapBuilder() + .add(LocationFeature, Option(locationDetails)) + .build() + } + .getOrElse { + DefaultFeatureMap + } + }.handle { + case e => + DefaultFeatureMap + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserHistoryTransformerEmbeddingQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserHistoryTransformerEmbeddingQueryFeatureHydrator.scala new file mode 100644 index 000000000..b83e1db26 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserHistoryTransformerEmbeddingQueryFeatureHydrator.scala @@ -0,0 +1,183 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.TransformerByteEmbeddingsAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.TransformerEmbeddingsAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.UserHistoryTransformerEmbeddingsHomeBlueAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.UserHistoryTransformerEmbeddingsHomeGreenAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.UserHistoryTransformerEmbeddingsJointBlueAdapter +import com.twitter.home_mixer_features.thriftscala.HomeMixerFeaturesType +import com.twitter.home_mixer_features.{thriftscala => hmf} +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.util.Future +import java.nio.ByteBuffer +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object UserHistoryTransformerEmbeddingHomeBlueFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} +object UserHistoryTransformerEmbeddingHomeGreenFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} +object UserHistoryTransformerEmbeddingJointBlueFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class UserHistoryTransformerEmbeddingQueryFeatureHydratorBuilder @Inject() ( + homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, + statsReceiver: StatsReceiver) { + def buildHomeBlueHydrator(): UserHistoryTransformerFloatEmbeddingQueryFeatureHydrator = { + UserHistoryTransformerFloatEmbeddingQueryFeatureHydrator( + FeatureHydratorIdentifier("HomeBlueTransformerEmbeddingQueryFeatureHydrator"), + homeMixerFeatureService, + statsReceiver, + UserHistoryTransformerEmbeddingHomeBlueFeature, + hmf.Cache.TransformerUserEmbeddings, + UserHistoryTransformerEmbeddingsHomeBlueAdapter + ) + } + + def buildHomeGreenHydrator(): UserHistoryTransformerByteEmbeddingQueryFeatureHydrator = { + UserHistoryTransformerByteEmbeddingQueryFeatureHydrator( + FeatureHydratorIdentifier("HomeGreenTransformerEmbeddingQueryFeatureHydrator"), + homeMixerFeatureService, + statsReceiver, + UserHistoryTransformerEmbeddingHomeGreenFeature, + hmf.Cache.TransformerUserEmbeddingsGreen, + UserHistoryTransformerEmbeddingsHomeGreenAdapter + ) + } + + def buildJointBlueHydrator(): UserHistoryTransformerByteEmbeddingQueryFeatureHydrator = { + UserHistoryTransformerByteEmbeddingQueryFeatureHydrator( + FeatureHydratorIdentifier("JointBlueTransformerEmbeddingQueryFeatureHydrator"), + homeMixerFeatureService, + statsReceiver, + UserHistoryTransformerEmbeddingJointBlueFeature, + hmf.Cache.TransformerUserJointEmbeddingsBlue, + UserHistoryTransformerEmbeddingsJointBlueAdapter + ) + } +} + +case class UserHistoryTransformerFloatEmbeddingQueryFeatureHydrator( + override val identifier: FeatureHydratorIdentifier, + override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, + override val statsReceiver: StatsReceiver, + override val embeddingFeature: DataRecordInAFeature[PipelineQuery], + override val cacheType: hmf.Cache, + transformerEmbeddingsAdapter: TransformerEmbeddingsAdapter) + extends BaseUserHistoryTransformerEmbeddingQueryFeatureHydrator { + + override def responseToDataRecord(homeMixerFeaturesType: HomeMixerFeaturesType): DataRecord = { + val embedding = homeMixerFeaturesType match { + case hmf.HomeMixerFeaturesType.RawEmbedding(floatEmbedding) => + floatEmbedding + case other => + wrongTypeCounter.incr() + throw new Exception( + f"Type not matching. Expected RawEmbedding but got ${other.getClass.getSimpleName}") + } + val tensor = ml.FloatTensor(embedding) + transformerEmbeddingsAdapter.adaptToDataRecords(Some(tensor)).asScala.head + } +} + +case class UserHistoryTransformerByteEmbeddingQueryFeatureHydrator( + override val identifier: FeatureHydratorIdentifier, + override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, + override val statsReceiver: StatsReceiver, + override val embeddingFeature: DataRecordInAFeature[PipelineQuery], + override val cacheType: hmf.Cache, + transformerByteEmbeddingsAdapter: TransformerByteEmbeddingsAdapter) + extends BaseUserHistoryTransformerEmbeddingQueryFeatureHydrator { + + override def responseToDataRecord(homeMixerFeaturesType: HomeMixerFeaturesType): DataRecord = { + val embedding = homeMixerFeaturesType match { + case hmf.HomeMixerFeaturesType.RawByteEmbedding(byteEmbedding) => + byteEmbedding + case other => + wrongTypeCounter.incr() + throw new Exception( + f"Type not matching. Expected RawByteEmbedding but got ${other.getClass.getSimpleName}") + } + val tensor = ml.RawTypedTensor(ml.DataType.Byte, convertToByteBuffer(embedding)) + transformerByteEmbeddingsAdapter.adaptToDataRecords(Some(tensor)).asScala.head + } + + private def convertToByteBuffer(byteArray: Seq[Byte]): ByteBuffer = { + val buffer = ByteBuffer.allocate(byteArray.size) + byteArray.foreach(buffer.put) + buffer.flip + buffer + } +} + +trait BaseUserHistoryTransformerEmbeddingQueryFeatureHydrator + extends QueryFeatureHydrator[PipelineQuery] { + + def homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint + def statsReceiver: StatsReceiver + def embeddingFeature: DataRecordInAFeature[PipelineQuery] + + def cacheType: hmf.Cache + + override val features: Set[Feature[_, _]] = Set(embeddingFeature) + + protected val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + protected val keyFoundCounter = scopedStatsReceiver.counter("key/found") + protected val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + protected val wrongTypeCounter = scopedStatsReceiver.counter("key/wrongType") + + def responseToDataRecord(homeMixerFeaturesType: HomeMixerFeaturesType): DataRecord + + private def getFeatureMap( + pipelineQuery: PipelineQuery + ): Future[FeatureMap] = { + val keysSerialized = Seq(pipelineQuery.getRequiredUserId.toString) + val request = hmf.HomeMixerFeaturesRequest(keysSerialized, cacheType) + val responseFut = + homeMixerFeatureService.getHomeMixerFeatures(request) + responseFut + .map { response => + response.homeMixerFeatures.headOption.flatMap { homeMixerFeatureOpt => + homeMixerFeatureOpt.homeMixerFeaturesType match { + case Some(homeMixerFeaturesType) => + keyFoundCounter.incr() + Some(responseToDataRecord(homeMixerFeaturesType)) + case None => + keyNotFoundCounter.incr() + None + } + } + }.handle { case _ => None } + .map { + case Some(dataRecord) => + FeatureMap(embeddingFeature, dataRecord) + case None => + FeatureMap(embeddingFeature, new DataRecord()) + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + Stitch.callFuture(getFeatureMap(query)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLanguagesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLanguagesFeatureHydrator.scala new file mode 100644 index 000000000..d4ca05f1d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLanguagesFeatureHydrator.scala @@ -0,0 +1,46 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesRepository +import com.twitter.home_mixer.util.ObservedKeyValueResultHandler +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.search.common.constants.{thriftscala => scc} +import com.twitter.servo.repository.KeyValueRepository +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +object UserLanguagesFeature extends Feature[PipelineQuery, Seq[scc.ThriftLanguage]] + +@Singleton +case class UserLanguagesFeatureHydrator @Inject() ( + @Named(UserLanguagesRepository) client: KeyValueRepository[Seq[Long], Long, Seq[ + scc.ThriftLanguage + ]], + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with ObservedKeyValueResultHandler { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserLanguages") + + override val features: Set[Feature[_, _]] = Set(UserLanguagesFeature) + + override val statScope: String = identifier.toString + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val key = query.getRequiredUserId + Stitch.callFuture(client(Seq(key))).map { result => + val feature = + observedGet(key = Some(key), keyValueResult = result).map(_.getOrElse(Seq.empty)) + FeatureMapBuilder() + .add(UserLanguagesFeature, feature) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLargeEmbeddingsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLargeEmbeddingsFeatureHydrator.scala new file mode 100644 index 000000000..cf17da693 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLargeEmbeddingsFeatureHydrator.scala @@ -0,0 +1,142 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.UserLargeEmbeddingsFeature +import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.UserLargeEmbeddingsKeyFeature +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam +import com.twitter.home_mixer_features.{thriftscala => hmf} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.adapters.large_embeddings.HashingFeatureParams +import com.twitter.timelines.prediction.adapters.large_embeddings.HomeMixerLargeEmbeddingsFeatureHydrator +import com.twitter.timelines.prediction.adapters.large_embeddings.LargeEmbeddingsAdapter +import com.twitter.timelines.prediction.adapters.large_embeddings.UserLargeEmbeddingsAdapter +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserLargeEmbeddingsFeatureHydrator @Inject() ( + statsReceiver: StatsReceiver, + override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] + with HomeMixerLargeEmbeddingsFeatureHydrator { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserLargeEmbeddings") + + override val features: Set[Feature[_, _]] = + Set(UserLargeEmbeddingsFeature, UserLargeEmbeddingsKeyFeature) + + override val adapter: LargeEmbeddingsAdapter = UserLargeEmbeddingsAdapter + + override val cacheType: hmf.Cache = hmf.Cache.UserLargeEmbeddings + + override val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableLargeEmbeddingsFeatureHydrationParam) + + // Hashing Features + override val defaultHashingFeatureParams: HashingFeatureParams = HashingFeatureParams( + scales = Seq(1681734645L, 1546314972L), + biases = Seq(2701200313L, 1873259806L), + modulus = 3055559939L, + bucketSize = 10000000L, + ) + + override val modelName2HashingFeatureParams: Map[String, HashingFeatureParams] = Map( + "hr_video_prod__v3_realtime" -> HashingFeatureParams( + scales = Seq(1341131000L, 519927459L), + biases = Seq(2924993425L, 294422133L), + modulus = 3109207999L, + bucketSize = 10000000L, + ), + "hr_video_prod__v2_lembeds" -> HashingFeatureParams( + scales = Seq(214226227L, 561611689L), + biases = Seq(182790211L, 330327483L), + modulus = 816016163L, + bucketSize = 1000000L, + ), + "hr_prod__v4_embeds_230M" -> HashingFeatureParams( + scales = Seq(196742702L, 1852108266L), + biases = Seq(1935840681L, 167407236L), + modulus = 2859568897L, + bucketSize = 100000000L, + ), + "hr_prod__v5_embeds_230M_and_transformer" -> HashingFeatureParams( + scales = Seq(196742702L, 1852108266L), + biases = Seq(1935840681L, 167407236L), + modulus = 2859568897L, + bucketSize = 100000000L, + ), + "hr_prod__v5_watchtime" -> HashingFeatureParams( + scales = Seq(45230244L, 676046872L), + biases = Seq(866394657L, 1019127517L), + modulus = 1047809363L, + bucketSize = 100000000L, + ), + "hr_prod__v6_transformer_v2" -> HashingFeatureParams( + scales = Seq(196742702L, 1852108266L), + biases = Seq(1935840681L, 167407236L), + modulus = 2859568897L, + bucketSize = 100000000L, + ), + "hr_prod__v6_mixed_training" -> HashingFeatureParams( + scales = Seq(196742702L, 1852108266L), + biases = Seq(1935840681L, 167407236L), + modulus = 2859568897L, + bucketSize = 100000000L, + ), + "hr_prod__v6_transformer_v2_kafka_merge_join" -> HashingFeatureParams( + scales = Seq(196742702L, 1852108266L), + biases = Seq(1935840681L, 167407236L), + modulus = 2859568897L, + bucketSize = 100000000L, + ), + "hr_prod__v6_transformer_v2_realtime_debias_21apr" -> HashingFeatureParams( + scales = Seq(196742702L, 1852108266L), + biases = Seq(1935840681L, 167407236L), + modulus = 2859568897L, + bucketSize = 100000000L, + ), + "hr_video_prod__v4_realtime" -> HashingFeatureParams( + scales = Seq(1341131000L, 519927459L), + biases = Seq(2924993425L, 294422133L), + modulus = 3109207999L, + bucketSize = 10000000L, + ), + "hr_video_prod__v4_realtime_mergehead" -> HashingFeatureParams( + scales = Seq(1341131000L, 519927459L), + biases = Seq(2924993425L, 294422133L), + modulus = 3109207999L, + bucketSize = 10000000L, + ), + ) + + private def getFeatureMap( + pipelineQuery: PipelineQuery + ): Future[FeatureMap] = { + val userId = pipelineQuery.getRequiredUserId + val modelName = pipelineQuery.params(ModelNameParam) + val responseMap = getLargeEmbeddings(userId, modelName) + responseMap.map { response => + FeatureMapBuilder() + .add(UserLargeEmbeddingsFeature, response.dataRecord) + .add(UserLargeEmbeddingsKeyFeature, response.hashedKeys) + .build() + } + } + + override def hydrate( + query: PipelineQuery + ): Stitch[FeatureMap] = Stitch.callFuture(getFeatureMap(query)) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserStateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserStateQueryFeatureHydrator.scala new file mode 100644 index 000000000..548a1682a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserStateQueryFeatureHydrator.scala @@ -0,0 +1,52 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelines.user_health.v1.{thriftscala => uhv1} +import com.twitter.timelines.user_health.{thriftscala => uh} +import com.twitter.user_session_store.ReadOnlyUserSessionStore +import com.twitter.user_session_store.ReadRequest +import com.twitter.user_session_store.UserSessionDataset +import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class UserStateQueryFeatureHydrator @Inject() ( + userSessionStore: ReadOnlyUserSessionStore) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserState") + + override val features: Set[Feature[_, _]] = Set(UserStateFeature) + + private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.UserHealth) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + userSessionStore + .read(ReadRequest(query.getRequiredUserId, datasets)) + .map { userSession => + val userState = userSession.flatMap { + _.userHealth match { + case Some(uh.UserHealth.V1(uhv1.UserHealth(userState))) => userState + case _ => None + } + } + + FeatureMapBuilder() + .add(UserStateFeature, userState) + .build() + } + } + + override val alerts = Seq( + HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99) + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserSubscriptionQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserSubscriptionQueryFeatureHydrator.scala new file mode 100644 index 000000000..ca448ab9c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserSubscriptionQueryFeatureHydrator.scala @@ -0,0 +1,37 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.subscription_services.subscription_verification.HasNoAdsBenefitOnUserClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +object NoAdsTierFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Boolean] { + override val defaultValue: Boolean = false +} + +@Singleton +case class UserSubscriptionQueryFeatureHydrator @Inject() ( + hasNoAdsBenefitOnUserClientColumn: HasNoAdsBenefitOnUserClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserSubscription") + + override val features: Set[Feature[_, _]] = Set(NoAdsTierFeature) + + override def hydrate( + query: PipelineQuery + ): Stitch[FeatureMap] = hasNoAdsBenefitOnUserClientColumn.fetcher + .fetch(query.getRequiredUserId) + .map { result => + FeatureMapBuilder() + .add(NoAdsTierFeature, result.v.getOrElse(false)) + .build() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserUnderstandableLangaugesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserUnderstandableLangaugesFeatureHydrator.scala new file mode 100644 index 000000000..2fbc266d8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserUnderstandableLangaugesFeatureHydrator.scala @@ -0,0 +1,57 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.UserUnderstandableLanguagesFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableGrokAutoTranslateLanguageFilter +import com.twitter.home_mixer.util.ObservedKeyValueResultHandler +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.language.user.GrokAutoTranslateUnderstandableLanguagesOnUserClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserUnderstandableLanguagesFeatureHydrator @Inject() ( + understandableLanguagesClientColumn: GrokAutoTranslateUnderstandableLanguagesOnUserClientColumn, + override val statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] + with ObservedKeyValueResultHandler { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserUnderstandableLanguages") + + override val features: Set[Feature[_, _]] = Set(UserUnderstandableLanguagesFeature) + + override val statScope: String = identifier.toString + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val successCounter = scopedStatsReceiver.counter("success") + private val failedCounter = scopedStatsReceiver.counter("failure") + + private val DefaultFeatureMap = FeatureMap(UserUnderstandableLanguagesFeature, Seq.empty) + + private val fetcher: Fetcher[Long, Unit, Seq[String]] = + understandableLanguagesClientColumn.fetcher + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableGrokAutoTranslateLanguageFilter) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val key = query.getRequiredUserId + fetcher + .fetch(key, ()).map { result => + successCounter.incr() + FeatureMap(UserUnderstandableLanguagesFeature, result.v.getOrElse(Seq.empty)) + }.rescue { + case _ => + failedCounter.incr() + Stitch.value(DefaultFeatureMap) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UtegFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UtegFeatureHydrator.scala new file mode 100644 index 000000000..4c4188772 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UtegFeatureHydrator.scala @@ -0,0 +1,102 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.FavoritedByCountFeature +import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature +import com.twitter.home_mixer.model.HomeFeatures.RepliedByCountFeature +import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.RetweetedByCountFeature +import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.recos.recos_common.{thriftscala => rc} +import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} +import com.twitter.servo.keyvalue.KeyValueResult +import com.twitter.servo.repository.KeyValueRepository +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class UtegFeatureHydrator @Inject() ( + @Named(UtegSocialProofRepository) client: KeyValueRepository[ + (Seq[Long], (Long, Map[Long, Double])), + Long, + uteg.TweetRecommendation + ]) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Uteg") + + override val features: Set[Feature[_, _]] = Set( + FavoritedByUserIdsFeature, + RetweetedByEngagerIdsFeature, + RepliedByEngagerIdsFeature, + FavoritedByCountFeature, + RetweetedByCountFeature, + RepliedByCountFeature + ) + + override def onlyIf(query: PipelineQuery): Boolean = query.features + .exists(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double]).nonEmpty) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val seedUserWeights = query.features.map(_.get(RealGraphInNetworkScoresFeature)).get + + val sourceTweetIds = candidates.flatMap(_.features.getOrElse(SourceTweetIdFeature, None)) + val inReplyToTweetIds = candidates.flatMap(_.features.getOrElse(InReplyToTweetIdFeature, None)) + val tweetIds = candidates.map(_.candidate.id) + val tweetIdsToSend = (tweetIds ++ sourceTweetIds ++ inReplyToTweetIds).distinct + + val utegQuery = (tweetIdsToSend, (query.getRequiredUserId, seedUserWeights)) + + client(utegQuery).map(handleResponse(candidates, _)) + } + + private def handleResponse( + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + results: KeyValueResult[Long, uteg.TweetRecommendation], + ): Seq[FeatureMap] = { + candidates.map { candidate => + val inNetwork = candidate.features.getOrElse(FromInNetworkSourceFeature, false) + val candidateProof = results(candidate.candidate.id).toOption.flatten + val sourceProof = candidate.features + .getOrElse(SourceTweetIdFeature, None).flatMap(results(_).toOption.flatten) + val proofs = Seq(candidateProof, sourceProof).flatten.map(_.socialProofByType) + + val favoritedBy = proofs.flatMap(_.get(rc.SocialProofType.Favorite)).flatten + val retweetedBy = proofs.flatMap(_.get(rc.SocialProofType.Retweet)).flatten + val repliedBy = proofs.flatMap(_.get(rc.SocialProofType.Reply)).flatten + + val (favoritedByCount, retweetedByCount, repliedByCount) = + if (!inNetwork) { + (favoritedBy.size.toDouble, retweetedBy.size.toDouble, repliedBy.size.toDouble) + } else { (0.0, 0.0, 0.0) } + + FeatureMapBuilder(sizeHint = 6) + .add(FavoritedByUserIdsFeature, favoritedBy) + .add(RetweetedByEngagerIdsFeature, retweetedBy) + .add(RepliedByEngagerIdsFeature, repliedBy) + .add(FavoritedByCountFeature, favoritedByCount) + .add(RetweetedByCountFeature, retweetedByCount) + .add(RepliedByCountFeature, repliedByCount) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/VideoSummaryEmbeddingFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/VideoSummaryEmbeddingFeatureHydrator.scala new file mode 100644 index 000000000..ccb0a9be5 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/VideoSummaryEmbeddingFeatureHydrator.scala @@ -0,0 +1,83 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.VideoSummaryEmbeddingFeaturesAdaptor +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableVideoSummaryEmbeddingFeatureDeciderParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.VideoEmbeddingMHStore +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import com.twitter.media_understanding.video_summary.thriftscala.VideoEmbedding +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.util.logging.Logging +import scala.collection.JavaConverters._ + +object VideoSummaryEmbeddingFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class VideoSummaryEmbeddingFeatureHydrator @Inject() ( + @Named(VideoEmbeddingMHStore) videoEmbeddingStore: ReadableStore[Long, VideoEmbedding], + statsReceiver: StatsReceiver) + extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] + with Logging { + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "VideoSummaryEmbedding") + + override val features: Set[Feature[_, _]] = Set(VideoSummaryEmbeddingFeature) + + private val DefaultFeatureMap = + FeatureMapBuilder().add(VideoSummaryEmbeddingFeature, new DataRecord()).build() + + private val scopedCounter = statsReceiver.scope("VideoSummaryEmbeddingHydration") + private val nonEmptyCounter = scopedCounter.counter("nonEmpty") + private val emptyCounter = scopedCounter.counter("empty") + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableVideoSummaryEmbeddingFeatureDeciderParam) + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + existingFeatures: FeatureMap + ): Stitch[FeatureMap] = { + getFirstMediaId(existingFeatures).fold(Stitch.value(DefaultFeatureMap)) { mediaId => + Stitch + .callFuture(videoEmbeddingStore.get(mediaId)).map { resultOpt => + resultOpt match { + case Some(embedding) => + val dataRecord = VideoSummaryEmbeddingFeaturesAdaptor + .adaptToDataRecords(embedding.embedding).asScala.head + nonEmptyCounter.incr() + FeatureMapBuilder().add(VideoSummaryEmbeddingFeature, dataRecord).build() + case None => + emptyCounter.incr() + DefaultFeatureMap + } + }.onFailure { e => + error(s"Error fetching VideoSummaryEmbedding: $e") + Stitch.value(DefaultFeatureMap) + } + } + } + + private def getFirstMediaId(featureMap: FeatureMap): Option[Long] = + featureMap.getOrElse(TweetMediaIdsFeature, Seq.empty[Long]).headOption +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ViewCountsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ViewCountsFeatureHydrator.scala new file mode 100644 index 000000000..e277c1245 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ViewCountsFeatureHydrator.scala @@ -0,0 +1,69 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.ViewCountFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.catalog.Fetch +import com.twitter.strato.generated.client.viewcounts.ViewCountOnTweetClientColumn +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ViewCountsFeatureHydrator @Inject() ( + viewCountsColumn: ViewCountOnTweetClientColumn, + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ViewCounts") + + override val features: Set[Feature[_, _]] = + Set(ViewCountFeature) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") + + private val DefaultFeatureMap = FeatureMap(ViewCountFeature, None) + private val batchSize = 64 + + def getFeatureMaps( + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + ): Future[Seq[FeatureMap]] = { + val featureMapStitch = Stitch.traverse(candidates) { candidate => + viewCountsColumn.fetcher + .fetch(candidate.candidate.id, Unit) + .map { + case Fetch.Result(response, _) => + if (response.nonEmpty) keyFoundCounter.incr() + else keyNotFoundCounter.incr() + FeatureMap(ViewCountFeature, response) + case _ => + keyFailureCounter.incr() + DefaultFeatureMap + } + } + Stitch.run(featureMapStitch) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + OffloadFuturePools.offloadBatchSeqToFutureSeq( + candidates, + getFeatureMaps, + batchSize, + offload = true) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ViralContentCreatorMetricsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ViralContentCreatorMetricsFeatureHydrator.scala new file mode 100644 index 000000000..f75a57a8e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ViralContentCreatorMetricsFeatureHydrator.scala @@ -0,0 +1,42 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ViralContentCreatorFeature +import com.twitter.home_mixer.module.ViralContentCreatorsConfig +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Track metrics on how often we serve posts from viral content creators + */ +@Singleton +case class ViralContentCreatorMetricsFeatureHydrator @Inject() ( + viralContentCreatorsConfig: ViralContentCreatorsConfig) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ViralContentCreatorMetrics") + + override val features: Set[Feature[_, _]] = Set(ViralContentCreatorFeature) + + private val ViralContentCreators = viralContentCreatorsConfig.creators + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { + candidates.map { candidate => + val authorIdOpt = candidate.features.getOrElse(AuthorIdFeature, None) + FeatureMap(ViralContentCreatorFeature, authorIdOpt.exists(ViralContentCreators.contains)) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/WithDefaultFeatureMap.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/WithDefaultFeatureMap.scala new file mode 100644 index 000000000..1b6810458 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/WithDefaultFeatureMap.scala @@ -0,0 +1,8 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap + +trait WithDefaultFeatureMap { + // Make sure that default Feature Map has same features as defined in the feature hydrator + val defaultFeatureMap: FeatureMap +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala new file mode 100644 index 000000000..4c31fae79 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala @@ -0,0 +1,92 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features + +import com.twitter.home_mixer.util.DataRecordUtil +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.util.CompactDataRecordConverter +import com.twitter.ml.api.util.FDsl._ +import com.twitter.timelines.author_features.v1.{thriftjava => af} +import com.twitter.timelines.prediction.common.adapters.TimelinesAdapterBase +import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig +import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures +import scala.collection.JavaConverters._ + +object AuthorFeaturesAdapter extends TimelinesAdapterBase[af.AuthorFeatures] { + + private val Prefix = "original_author.timelines.original_author_aggregates." + + private val typedAggregateGroups = + TimelinesAggregationConfig.originalAuthorAggregatesV1.buildTypedAggregateGroups() + + private val aggregateFeaturesRenameMap: Map[Feature[_], Feature[_]] = + typedAggregateGroups.map(_.outputFeaturesToRenamedOutputFeatures(Prefix)).reduce(_ ++ _) + + private val prefixedOriginalAuthorAggregateFeatures = + typedAggregateGroups.flatMap(_.allOutputFeatures).map { feature => + aggregateFeaturesRenameMap.getOrElse(feature, feature) + } + + private val authorFeatures = prefixedOriginalAuthorAggregateFeatures ++ Seq( + UserHealthFeatures.AuthorState, + UserHealthFeatures.NumAuthorFollowers, + UserHealthFeatures.NumAuthorConnectDays, + UserHealthFeatures.NumAuthorConnect + ) + + private val aggregateFeatureContext: FeatureContext = + new FeatureContext(typedAggregateGroups.flatMap(_.allOutputFeatures).asJava) + + private lazy val prefixedAggregateFeatureContext: FeatureContext = + new FeatureContext(prefixedOriginalAuthorAggregateFeatures.asJava) + + override val getFeatureContext: FeatureContext = new FeatureContext(authorFeatures: _*) + + override val commonFeatures: Set[Feature[_]] = Set.empty + + private val compactDataRecordConverter = new CompactDataRecordConverter() + + override def adaptToDataRecords( + authorFeatures: af.AuthorFeatures + ): java.util.List[DataRecord] = { + val dataRecord = + if (authorFeatures.aggregates != null) { + val originalAuthorAggregatesDataRecord = + compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates) + + DataRecordUtil.applyRename( + originalAuthorAggregatesDataRecord, + aggregateFeatureContext, + prefixedAggregateFeatureContext, + aggregateFeaturesRenameMap) + } else new DataRecord + + if (authorFeatures.user_health != null) { + val userHealth = authorFeatures.user_health + + if (userHealth.user_state != null) { + dataRecord.setFeatureValue( + UserHealthFeatures.AuthorState, + userHealth.user_state.getValue.toLong + ) + } + + dataRecord.setFeatureValue( + UserHealthFeatures.NumAuthorFollowers, + userHealth.num_followers.toDouble + ) + + dataRecord.setFeatureValue( + UserHealthFeatures.NumAuthorConnectDays, + userHealth.num_connect_days.toDouble + ) + + dataRecord.setFeatureValue( + UserHealthFeatures.NumAuthorConnect, + userHealth.num_connect.toDouble + ) + } + + List(dataRecord).asJava + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/BUILD.bazel new file mode 100644 index 000000000..08f551b97 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/BUILD.bazel @@ -0,0 +1,17 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "src/java/com/twitter/ml/api:api-base", + "src/java/com/twitter/ml/api/util", + "src/scala/com/twitter/ml/api/util", + "src/scala/com/twitter/timelines/prediction/common/adapters:base", + "src/scala/com/twitter/timelines/prediction/common/aggregates", + "src/scala/com/twitter/timelines/prediction/features/user_health", + "src/thrift/com/twitter/timelines/author_features:thrift-java", + "timelines/data_processing/ml_util/aggregation_framework", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/BUILD.bazel new file mode 100644 index 000000000..ccfbfffd4 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/BUILD.bazel @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "src/java/com/twitter/ml/api:api-base", + "src/scala/com/twitter/ml/api/util", + "src/scala/com/twitter/timelines/prediction/common/adapters", + "src/scala/com/twitter/timelines/prediction/features/common", + "src/scala/com/twitter/timelines/prediction/features/conversation_features", + "src/scala/com/twitter/timelines/prediction/features/recap", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ClipEmbeddingFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ClipEmbeddingFeaturesAdapter.scala new file mode 100644 index 000000000..086fec29b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ClipEmbeddingFeaturesAdapter.scala @@ -0,0 +1,31 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content + +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.FloatTensor +import com.twitter.ml.api.GeneralTensor +import com.twitter.ml.api.RichDataRecord +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import scala.collection.JavaConverters._ + +object ClipEmbeddingFeaturesAdapter extends TimelinesMutatingAdapterBase[Seq[Double]] { + + val ClipEmbeddingsFeature: Feature.Tensor = TimelinesSharedFeatures.CLIP_EMBEDDING + + override val getFeatureContext: FeatureContext = new FeatureContext(ClipEmbeddingsFeature) + + override val commonFeatures: Set[Feature[_]] = Set.empty + + override def setFeatures( + clipEmbedding: Seq[Double], + richDataRecord: RichDataRecord + ): Unit = { + val clipEmbeddingTensor = new GeneralTensor() + clipEmbeddingTensor.setFloatTensor(new FloatTensor(clipEmbedding.map(Double.box).asJava)) + richDataRecord.setFeatureValue( + ClipEmbeddingsFeature, + clipEmbeddingTensor + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ContentFeatureAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ContentFeatureAdapter.scala new file mode 100644 index 000000000..08dbeba5e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ContentFeatureAdapter.scala @@ -0,0 +1,284 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content + +import com.twitter.home_mixer.model.ContentFeatures +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.ml.api.util.DataRecordConverters._ +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.common.adapters.TweetLengthType +import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import com.twitter.timelines.prediction.features.conversation_features.ConversationFeatures +import com.twitter.timelines.prediction.features.recap.RecapFeatures +import scala.collection.JavaConverters._ + +object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] { + + override val getFeatureContext: FeatureContext = new FeatureContext( + ConversationFeatures.IS_SELF_THREAD_TWEET, + ConversationFeatures.IS_LEAF_IN_SELF_THREAD, + TimelinesSharedFeatures.ASPECT_RATIO_DEN, + TimelinesSharedFeatures.ASPECT_RATIO_NUM, + TimelinesSharedFeatures.BIT_RATE, + TimelinesSharedFeatures.CLASSIFICATION_LABELS, + TimelinesSharedFeatures.COLOR_1_BLUE, + TimelinesSharedFeatures.COLOR_1_GREEN, + TimelinesSharedFeatures.COLOR_1_PERCENTAGE, + TimelinesSharedFeatures.COLOR_1_RED, + TimelinesSharedFeatures.FACE_AREAS, + TimelinesSharedFeatures.HAS_APP_INSTALL_CALL_TO_ACTION, + TimelinesSharedFeatures.HAS_DESCRIPTION, + TimelinesSharedFeatures.HAS_QUESTION, + TimelinesSharedFeatures.HAS_SELECTED_PREVIEW_IMAGE, + TimelinesSharedFeatures.HAS_TITLE, + TimelinesSharedFeatures.HAS_VISIT_SITE_CALL_TO_ACTION, + TimelinesSharedFeatures.HAS_WATCH_NOW_CALL_TO_ACTION, + TimelinesSharedFeatures.HEIGHT_1, + TimelinesSharedFeatures.HEIGHT_2, + TimelinesSharedFeatures.HEIGHT_3, + TimelinesSharedFeatures.HEIGHT_4, + TimelinesSharedFeatures.IS_360, + TimelinesSharedFeatures.IS_EMBEDDABLE, + TimelinesSharedFeatures.IS_MANAGED, + TimelinesSharedFeatures.IS_MONETIZABLE, + TimelinesSharedFeatures.MEDIA_PROVIDERS, + TimelinesSharedFeatures.NUM_CAPS, + TimelinesSharedFeatures.NUM_COLOR_PALLETTE_ITEMS, + TimelinesSharedFeatures.NUM_FACES, + TimelinesSharedFeatures.NUM_MEDIA_TAGS, + TimelinesSharedFeatures.NUM_NEWLINES, + TimelinesSharedFeatures.NUM_STICKERS, + TimelinesSharedFeatures.NUM_WHITESPACES, + TimelinesSharedFeatures.RESIZE_METHOD_1, + TimelinesSharedFeatures.RESIZE_METHOD_2, + TimelinesSharedFeatures.RESIZE_METHOD_3, + TimelinesSharedFeatures.RESIZE_METHOD_4, + TimelinesSharedFeatures.TWEET_LENGTH, + TimelinesSharedFeatures.TWEET_LENGTH_TYPE, + TimelinesSharedFeatures.VIDEO_DURATION, + TimelinesSharedFeatures.VIEW_COUNT, + TimelinesSharedFeatures.WIDTH_1, + TimelinesSharedFeatures.WIDTH_2, + TimelinesSharedFeatures.WIDTH_3, + TimelinesSharedFeatures.WIDTH_4, + RecapFeatures.HAS_VIDEO, + RecapFeatures.HAS_IMAGE, + ) + + override val commonFeatures: Set[Feature[_]] = Set.empty + + private def getTweetLengthType(tweetLength: Int): Long = { + tweetLength match { + case x if 0 > x || 280 < x => TweetLengthType.INVALID + case x if 0 <= x && x <= 30 => TweetLengthType.VERY_SHORT + case x if 30 < x && x <= 60 => TweetLengthType.SHORT + case x if 60 < x && x <= 90 => TweetLengthType.MEDIUM + case x if 90 < x && x <= 140 => TweetLengthType.LENGTHY + case x if 140 < x && x <= 210 => TweetLengthType.VERY_LENGTHY + case x if x > 210 => TweetLengthType.MAXIMUM_LENGTH + } + } + + override def setFeatures( + contentFeatures: Option[ContentFeatures], + richDataRecord: RichDataRecord + ): Unit = { + if (contentFeatures.nonEmpty) { + val features = contentFeatures.get + // Conversation Features + richDataRecord.setFeatureValueFromOption( + ConversationFeatures.IS_SELF_THREAD_TWEET, + Some(features.selfThreadMetadata.nonEmpty) + ) + richDataRecord.setFeatureValueFromOption( + ConversationFeatures.IS_LEAF_IN_SELF_THREAD, + features.selfThreadMetadata.map(_.isLeaf) + ) + + // Media Features + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.ASPECT_RATIO_DEN, + features.aspectRatioDen.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.ASPECT_RATIO_NUM, + features.aspectRatioNum.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.BIT_RATE, + features.bitRate.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.HEIGHT_1, + features.heights.flatMap(_.lift(0)).map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.HEIGHT_2, + features.heights.flatMap(_.lift(1)).map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.HEIGHT_3, + features.heights.flatMap(_.lift(2)).map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.HEIGHT_4, + features.heights.flatMap(_.lift(3)).map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.NUM_MEDIA_TAGS, + features.numMediaTags.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.RESIZE_METHOD_1, + features.resizeMethods.flatMap(_.lift(0)).map(_.toLong) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.RESIZE_METHOD_2, + features.resizeMethods.flatMap(_.lift(1)).map(_.toLong) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.RESIZE_METHOD_3, + features.resizeMethods.flatMap(_.lift(2)).map(_.toLong) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.RESIZE_METHOD_4, + features.resizeMethods.flatMap(_.lift(3)).map(_.toLong) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.VIDEO_DURATION, + features.videoDurationMs.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.WIDTH_1, + features.widths.flatMap(_.lift(0)).map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.WIDTH_2, + features.widths.flatMap(_.lift(1)).map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.WIDTH_3, + features.widths.flatMap(_.lift(2)).map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.WIDTH_4, + features.widths.flatMap(_.lift(3)).map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.NUM_COLOR_PALLETTE_ITEMS, + features.numColors.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.COLOR_1_RED, + features.dominantColorRed.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.COLOR_1_BLUE, + features.dominantColorBlue.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.COLOR_1_GREEN, + features.dominantColorGreen.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.COLOR_1_PERCENTAGE, + features.dominantColorPercentage + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.MEDIA_PROVIDERS, + features.mediaOriginProviders.map(_.toSet.asJava) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.IS_360, + features.is360 + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.VIEW_COUNT, + features.viewCount.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.IS_MANAGED, + features.isManaged + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.IS_MONETIZABLE, + features.isMonetizable + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.IS_EMBEDDABLE, + features.isEmbeddable + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.NUM_STICKERS, + features.stickerIds.map(_.length.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.NUM_FACES, + features.faceAreas.map(_.length.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.FACE_AREAS, + // guard for exception from max on empty seq + features.faceAreas.map(faceAreas => + faceAreas.map(_.toDouble).reduceOption(_ max _).getOrElse(0.0)) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.HAS_SELECTED_PREVIEW_IMAGE, + features.hasSelectedPreviewImage + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.HAS_TITLE, + features.hasTitle + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.HAS_DESCRIPTION, + features.hasDescription + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.HAS_VISIT_SITE_CALL_TO_ACTION, + features.hasVisitSiteCallToAction + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.HAS_APP_INSTALL_CALL_TO_ACTION, + features.hasAppInstallCallToAction + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.HAS_WATCH_NOW_CALL_TO_ACTION, + features.hasWatchNowCallToAction + ) + // text features + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.NUM_CAPS, + Some(features.numCaps.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.TWEET_LENGTH, + Some(features.length.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.TWEET_LENGTH_TYPE, + Some(getTweetLengthType(features.length.toInt)) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.NUM_WHITESPACES, + Some(features.numWhiteSpaces.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.HAS_QUESTION, + Some(features.hasQuestion) + ) + richDataRecord.setFeatureValueFromOption( + TimelinesSharedFeatures.NUM_NEWLINES, + features.numNewlines.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + RecapFeatures.HAS_IMAGE, + features.hasImage + ) + richDataRecord.setFeatureValueFromOption( + RecapFeatures.HAS_VIDEO, + features.hasVideo + ) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/TextTokensFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/TextTokensFeaturesAdapter.scala new file mode 100644 index 000000000..25495dddf --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/TextTokensFeaturesAdapter.scala @@ -0,0 +1,31 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content + +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.ml.api.thriftscala.Int32Tensor +import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import com.twitter.ml.api.{thriftscala => ml} + +object TextTokensFeaturesAdapter extends TimelinesMutatingAdapterBase[Seq[Int]] { + + private val TextTokenFeature: Feature.Tensor = + TimelinesSharedFeatures.TEXT_TOKENS_EMBEDDING + + override val getFeatureContext: FeatureContext = new FeatureContext(TextTokenFeature) + + override val commonFeatures: Set[Feature[_]] = Set.empty + + override def setFeatures( + textTokens: Seq[Int], + richDataRecord: RichDataRecord + ): Unit = { + richDataRecord.setFeatureValue( + TextTokenFeature, + ScalaToJavaDataRecordConversions.scalaTensor2Java( + ml.GeneralTensor.Int32Tensor(Int32Tensor(ints = textTokens)) + )) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/VideoSummaryEmbeddingFeaturesAdaptor.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/VideoSummaryEmbeddingFeaturesAdaptor.scala new file mode 100644 index 000000000..4e5506fda --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/VideoSummaryEmbeddingFeaturesAdaptor.scala @@ -0,0 +1,32 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content + +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.FloatTensor +import com.twitter.ml.api.RichDataRecord +import com.twitter.ml.api.GeneralTensor +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import scala.collection.JavaConverters._ + +object VideoSummaryEmbeddingFeaturesAdaptor extends TimelinesMutatingAdapterBase[Seq[Double]] { + + val VideoSummaryEmbeddingFeature: Feature.Tensor = TimelinesSharedFeatures.VIDEO_SUMMARY_EMBEDDING + + override val getFeatureContext: FeatureContext = new FeatureContext(VideoSummaryEmbeddingFeature) + + override val commonFeatures: Set[Feature[_]] = Set.empty + + override def setFeatures( + videoSummaryEmbedding: Seq[Double], + richDataRecord: RichDataRecord + ): Unit = { + val summaryEmbeddingTensor = new GeneralTensor() + summaryEmbeddingTensor.setFloatTensor( + new FloatTensor(videoSummaryEmbedding.map(Double.box).asJava)) + richDataRecord.setFeatureValue( + VideoSummaryEmbeddingFeature, + summaryEmbeddingTensor + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features/BUILD.bazel new file mode 100644 index 000000000..53133152f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features/BUILD.bazel @@ -0,0 +1,11 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "src/java/com/twitter/ml/api:api-base", + "src/scala/com/twitter/timelines/prediction/common/adapters:base", + "src/scala/com/twitter/timelines/prediction/features/gizmoduck", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features/GizmoduckFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features/GizmoduckFeaturesAdapter.scala new file mode 100644 index 000000000..156bfa5bc --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features/GizmoduckFeaturesAdapter.scala @@ -0,0 +1,50 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.gizmoduck_features + +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.features.gizmoduck.GizmoduckFeatures +import java.lang.{Boolean => JBoolean} + +object GizmoduckFeaturesAdapter extends TimelinesMutatingAdapterBase[GFeatures] { + + override val getFeatureContext: FeatureContext = new FeatureContext( + GizmoduckFeatures.AUTHOR_IS_BLUE_VERIFIED, + GizmoduckFeatures.AUTHOR_IS_VERIFIED_ORGANIZATION, + GizmoduckFeatures.AUTHOR_IS_VERIFIED_ORGANIZATION_AFFILIATE, + GizmoduckFeatures.AUTHOR_IS_PROTECTED, + ) + + override val commonFeatures: Set[Feature[_]] = Set.empty + + override def candidateFeatures: Set[Feature[_]] = super.candidateFeatures + override def setFeatures( + contentFeatures: GFeatures, + richDataRecord: RichDataRecord + ): Unit = { + richDataRecord.setFeatureValue[JBoolean]( + GizmoduckFeatures.AUTHOR_IS_BLUE_VERIFIED, + contentFeatures.isBlueVerified + ) + richDataRecord.setFeatureValue[JBoolean]( + GizmoduckFeatures.AUTHOR_IS_VERIFIED_ORGANIZATION, + contentFeatures.isVerifiedOrganization + ) + richDataRecord.setFeatureValue[JBoolean]( + GizmoduckFeatures.AUTHOR_IS_VERIFIED_ORGANIZATION_AFFILIATE, + contentFeatures.isVerifiedOrganizationAffiliate + ) + richDataRecord.setFeatureValue[JBoolean]( + GizmoduckFeatures.AUTHOR_IS_PROTECTED, + contentFeatures.isProtected + ) + } +} + +trait GFeatures { + def isBlueVerified: Boolean + def isVerifiedOrganization: Boolean + def isVerifiedOrganizationAffiliate: Boolean + def isProtected: Boolean +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/BUILD.bazel new file mode 100644 index 000000000..82fd6fabb --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/BUILD.bazel @@ -0,0 +1,11 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "src/java/com/twitter/ml/api:api-base", + "src/scala/com/twitter/timelines/prediction/common/adapters:base", + "src/scala/com/twitter/timelines/prediction/features/common", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala new file mode 100644 index 000000000..62125ea5a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala @@ -0,0 +1,25 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic + +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import scala.collection.JavaConverters._ + +object InferredTopicAdapter extends TimelinesMutatingAdapterBase[Map[Long, Double]] { + + override val getFeatureContext: FeatureContext = new FeatureContext( + TimelinesSharedFeatures.INFERRED_TOPIC_IDS) + + override val commonFeatures: Set[Feature[_]] = Set.empty + + override def setFeatures( + inferredTopicFeatures: Map[Long, Double], + richDataRecord: RichDataRecord + ): Unit = { + richDataRecord.setFeatureValue( + TimelinesSharedFeatures.INFERRED_TOPIC_IDS, + inferredTopicFeatures.keys.map(_.toString).toSet.asJava) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features/BUILD.bazel new file mode 100644 index 000000000..72f9fb4ec --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features/BUILD.bazel @@ -0,0 +1,12 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/thrift/src/main/thrift:thrift-scala", + "src/java/com/twitter/ml/api:api-base", + "src/scala/com/twitter/timelines/prediction/common/adapters:base", + "src/scala/com/twitter/timelines/prediction/features/common", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features/LightRankingCandidateFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features/LightRankingCandidateFeaturesAdapter.scala new file mode 100644 index 000000000..bf0c47fb7 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features/LightRankingCandidateFeaturesAdapter.scala @@ -0,0 +1,64 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.light_ranking_features + +import com.twitter.home_mixer.thriftscala.ServedType +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import java.lang.{Boolean => JBoolean} +import java.lang.{Long => JLong} +import java.lang.{String => JString} + +case class LightRankingCandidateFeatures( + isSelected: Boolean, + isSelectedByHeavyRanker: Boolean, + rankByHeavyRanker: Int, + servedType: ServedType, + candidateSourcePosition: Long, +) + +/** + * define light ranking features adapter to create a data record which includes many light ranking features + * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. + */ +object LightRankingCandidateFeaturesAdapter + extends TimelinesMutatingAdapterBase[LightRankingCandidateFeatures] { + + val featureContext = new FeatureContext( + TimelinesSharedFeatures.IS_SELECTED, + TimelinesSharedFeatures.IS_SELECTED_BY_HEAVY_RANKER, + TimelinesSharedFeatures.RANK_BY_HEAVY_RANKER, + TimelinesSharedFeatures.SERVED_TYPE_ID, + TimelinesSharedFeatures.SERVED_TYPE, + TimelinesSharedFeatures.CANDIDATE_SOURCE_POSITION, + ) + + override def getFeatureContext: FeatureContext = featureContext + + override val commonFeatures: Set[Feature[_]] = Set.empty + + override def setFeatures( + lightRankingCandidateFeatures: LightRankingCandidateFeatures, + richDataRecord: RichDataRecord + ): Unit = { + richDataRecord.setFeatureValue[JBoolean]( + TimelinesSharedFeatures.IS_SELECTED, + lightRankingCandidateFeatures.isSelected) + richDataRecord.setFeatureValue[JBoolean]( + TimelinesSharedFeatures.IS_SELECTED_BY_HEAVY_RANKER, + lightRankingCandidateFeatures.isSelectedByHeavyRanker) + richDataRecord.setFeatureValue[JLong]( + TimelinesSharedFeatures.RANK_BY_HEAVY_RANKER, + lightRankingCandidateFeatures.rankByHeavyRanker) + richDataRecord.setFeatureValue[JString]( + TimelinesSharedFeatures.SERVED_TYPE, + lightRankingCandidateFeatures.servedType.name) + richDataRecord.setFeatureValue[JLong]( + TimelinesSharedFeatures.SERVED_TYPE_ID, + lightRankingCandidateFeatures.servedType.getValue()) + richDataRecord.setFeatureValue[JLong]( + TimelinesSharedFeatures.CANDIDATE_SOURCE_POSITION, + lightRankingCandidateFeatures.candidateSourcePosition) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/BUILD.bazel new file mode 100644 index 000000000..a850b4db1 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/BUILD.bazel @@ -0,0 +1,13 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "src/java/com/twitter/ml/api:api-base", + "src/java/com/twitter/ml/api/constant", + "src/scala/com/twitter/timelines/prediction/common/adapters:base", + "src/scala/com/twitter/timelines/prediction/features/common", + "src/scala/com/twitter/timelines/prediction/features/request_context", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala new file mode 100644 index 000000000..2e4a22cb6 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala @@ -0,0 +1,45 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features + +import com.twitter.ml.api.constant.SharedFeatures +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import java.lang.{Long => JLong} + +case class NonMLCandidateFeatures( + tweetId: Long, + sourceTweetId: Long, + originalAuthorId: Option[Long], +) + +/** + * define non ml features adapter to create a data record which includes many non ml features + * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. + */ +object NonMLCandidateFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCandidateFeatures] { + + private val featureContext = new FeatureContext( + SharedFeatures.TWEET_ID, + // For Secondary Engagement data generation + TimelinesSharedFeatures.SOURCE_TWEET_ID, + TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, + ) + + override def getFeatureContext: FeatureContext = featureContext + + override val commonFeatures: Set[Feature[_]] = Set.empty + + override def setFeatures( + nonMLCandidateFeatures: NonMLCandidateFeatures, + richDataRecord: RichDataRecord + ): Unit = { + richDataRecord.setFeatureValue[JLong](SharedFeatures.TWEET_ID, nonMLCandidateFeatures.tweetId) + richDataRecord.setFeatureValue[JLong]( + TimelinesSharedFeatures.SOURCE_TWEET_ID, + nonMLCandidateFeatures.sourceTweetId) + nonMLCandidateFeatures.originalAuthorId.foreach( + richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, _)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala new file mode 100644 index 000000000..8e54d3ef1 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala @@ -0,0 +1,71 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features + +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.ml.api.constant.SharedFeatures +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import com.twitter.timelines.prediction.features.request_context.RequestContextFeatures +import java.lang.{Long => JLong} +import java.lang.{String => JString} + +case class NonMLCommonFeatures( + userId: Long, + guestId: Option[Long], + clientId: Option[Long], + countryCode: Option[String], + predictionRequestId: Option[Long], + productSurface: String, + servedTimestamp: Long, +) + +/** + * define non ml features adapter to create a data record which includes many non ml features + * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. + */ +object NonMLCommonFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCommonFeatures] { + + private val featureContext = new FeatureContext( + SharedFeatures.USER_ID, + SharedFeatures.GUEST_ID, + SharedFeatures.CLIENT_ID, + TimelinesSharedFeatures.PREDICTION_REQUEST_ID, + TimelinesSharedFeatures.PRODUCT_SURFACE, + TimelinesSharedFeatures.SERVED_TIMESTAMP, + RequestContextFeatures.COUNTRY_CODE, + ) + + override def getFeatureContext: FeatureContext = featureContext + + override val commonFeatures: Set[Feature[_]] = Set( + SharedFeatures.USER_ID, + SharedFeatures.GUEST_ID, + SharedFeatures.CLIENT_ID, + TimelinesSharedFeatures.PREDICTION_REQUEST_ID, + TimelinesSharedFeatures.PRODUCT_SURFACE, + TimelinesSharedFeatures.SERVED_TIMESTAMP, + RequestContextFeatures.COUNTRY_CODE, + ) + + override def setFeatures( + nonMLCommonFeatures: NonMLCommonFeatures, + richDataRecord: RichDataRecord + ): Unit = { + richDataRecord.setFeatureValue[JLong](SharedFeatures.USER_ID, nonMLCommonFeatures.userId) + nonMLCommonFeatures.guestId.foreach( + richDataRecord.setFeatureValue[JLong](SharedFeatures.GUEST_ID, _)) + nonMLCommonFeatures.clientId.foreach( + richDataRecord.setFeatureValue[JLong](SharedFeatures.CLIENT_ID, _)) + nonMLCommonFeatures.predictionRequestId.foreach( + richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.PREDICTION_REQUEST_ID, _)) + nonMLCommonFeatures.countryCode.foreach( + richDataRecord.setFeatureValue[JString](RequestContextFeatures.COUNTRY_CODE, _)) + richDataRecord.setFeatureValue( + TimelinesSharedFeatures.PRODUCT_SURFACE, + nonMLCommonFeatures.productSurface) + richDataRecord.setFeatureValue[JLong]( + TimelinesSharedFeatures.SERVED_TIMESTAMP, + nonMLCommonFeatures.servedTimestamp) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/BUILD.bazel new file mode 100644 index 000000000..d4923c8c3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/BUILD.bazel @@ -0,0 +1,12 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "src/java/com/twitter/ml/api:api-base", + "src/scala/com/twitter/timelines/prediction/common/adapters:base", + "src/thrift/com/twitter/ml/api:data-java", + "timelines/data_processing/ml_util/aggregation_framework/conversion:for-timelines", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala new file mode 100644 index 000000000..cd5ab020f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala @@ -0,0 +1,12 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates + +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.IRecordOneToOneAdapter + +object PassThroughAdapter extends IRecordOneToOneAdapter[Seq[DataRecord]] { + override def adaptToDataRecord(record: Seq[DataRecord]): DataRecord = + record.headOption.getOrElse(new DataRecord) + + // This is not necessary and should not be used. + override def getFeatureContext = ??? +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala new file mode 100644 index 000000000..4b894ae5f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala @@ -0,0 +1,17 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates + +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.conversion.CombineCountsPolicy +import com.twitter.timelines.prediction.common.adapters.TimelinesIRecordAdapter + +class SparseAggregatesToDenseAdapter(policy: CombineCountsPolicy) + extends TimelinesIRecordAdapter[Seq[DataRecord]] { + + override def setFeatures(input: Seq[DataRecord], mutableDataRecord: RichDataRecord): Unit = + policy.defaultMergeRecord(mutableDataRecord.getRecord, input) + + override val getFeatureContext: FeatureContext = + new FeatureContext(policy.outputFeaturesPostMerge.toSeq: _*) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features/BUILD.bazel new file mode 100644 index 000000000..146d824f4 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features/BUILD.bazel @@ -0,0 +1,12 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "src/java/com/twitter/ml/api:api-base", + "src/scala/com/twitter/timelines/prediction/common/adapters", + "src/scala/com/twitter/timelines/prediction/features/simcluster", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features/SimclustersFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features/SimclustersFeaturesAdapter.scala new file mode 100644 index 000000000..1e73ae20d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features/SimclustersFeaturesAdapter.scala @@ -0,0 +1,42 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.simclusters_features + +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.simclusters_v2.thriftscala.SimClustersEmbedding +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.features.simcluster.SimclusterFeatures +import java.util +import java.util.Collections + +object SimclustersFeaturesAdapter + extends TimelinesMutatingAdapterBase[Option[SimClustersEmbedding]] { + + val SimclustersSparseTweetEmbeddingsFeature: Feature.SparseContinuous = + SimclusterFeatures.SIMCLUSTER_TWEET_CLUSTER_SCORES + + override val getFeatureContext: FeatureContext = new FeatureContext( + SimclustersSparseTweetEmbeddingsFeature) + + override val commonFeatures: Set[Feature[_]] = Set.empty + + override def setFeatures( + simClustersEmbedding: Option[SimClustersEmbedding], + richDataRecord: RichDataRecord + ): Unit = { + val simclustersWithScoresMap = simClustersEmbedding match { + case Some(value) => + val result = new util.HashMap[String, java.lang.Double](value.embedding.size, 1.0f) + value.embedding.foreach { simclusterWithScore => + result.put(simclusterWithScore.clusterId.toString, simclusterWithScore.score) + } + result + case None => + Collections.emptyMap[String, java.lang.Double]() + } + richDataRecord.setFeatureValue( + SimclustersSparseTweetEmbeddingsFeature, + simclustersWithScoresMap + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/BUILD.bazel new file mode 100644 index 000000000..ea0e32782 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/BUILD.bazel @@ -0,0 +1,13 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "src/java/com/twitter/ml/api:api-base", + "src/scala/com/twitter/ml/api/util", + "src/scala/com/twitter/timelines/prediction/common/adapters:base", + "src/thrift/com/twitter/ml/api:data-java", + "user_history_transformer/thrift/src/main/thrift/com/twitter/user_history_transformer:user_history_transformer-scala", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/TransformerEmbeddingsAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/TransformerEmbeddingsAdapter.scala new file mode 100644 index 000000000..89945a5ae --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/TransformerEmbeddingsAdapter.scala @@ -0,0 +1,129 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings + +import com.twitter.ml.api.DataType +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import scala.collection.JavaConverters._ + +sealed trait TransformerEmbeddingsAdapter + extends TimelinesMutatingAdapterBase[Option[ml.FloatTensor]] { + def embeddingsFeature: Feature.Tensor + + override def getFeatureContext: FeatureContext = new FeatureContext( + embeddingsFeature + ) + + override def setFeatures( + embedding: Option[ml.FloatTensor], + richDataRecord: RichDataRecord + ): Unit = { + embedding.foreach { floatTensor => + richDataRecord.setFeatureValue( + embeddingsFeature, + ScalaToJavaDataRecordConversions.scalaTensor2Java( + ml.GeneralTensor + .FloatTensor(floatTensor))) + } + } +} + +sealed trait TransformerByteEmbeddingsAdapter + extends TimelinesMutatingAdapterBase[Option[ml.RawTypedTensor]] { + def embeddingsFeature: Feature.Tensor + + override def getFeatureContext: FeatureContext = new FeatureContext( + embeddingsFeature + ) + + override def setFeatures( + embedding: Option[ml.RawTypedTensor], + richDataRecord: RichDataRecord + ): Unit = { + embedding.foreach { tensor => + richDataRecord.setFeatureValue( + embeddingsFeature, + ScalaToJavaDataRecordConversions.scalaTensor2Java( + ml.GeneralTensor + .RawTypedTensor(tensor))) + } + } +} + +object TransformerEmbeddingsFeatures { + val UserHistoryTransformerEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "user.transformer.user_history_as_float_tensor", + DataType.FLOAT, + List(128L).map(long2Long).asJava, + ) + val UserHistoryTransformerEmbeddingsGreenFeature: Feature.Tensor = new Feature.Tensor( + "user.transformer.green.user_history_as_float_tensor", + DataType.BYTE, + List(1024L).map(long2Long).asJava, + ) + val UserHistoryTransformerEmbeddingsJointBlueFeature: Feature.Tensor = new Feature.Tensor( + "user.transformer.joint.blue.user_history_as_float_tensor", + DataType.BYTE, + List(1024L).map(long2Long).asJava, + ) + val PostTransformerEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "tweet_id.transformer.post_as_float_tensor", + DataType.FLOAT, + List(128L).map(long2Long).asJava, + ) + val PostTransformerEmbeddingsGreenFeature: Feature.Tensor = new Feature.Tensor( + "tweet_id.transformer.green.post_as_float_tensor", + DataType.FLOAT, + List(128L).map(long2Long).asJava, + ) + val PostTransformerEmbeddingsJointBlueFeature: Feature.Tensor = new Feature.Tensor( + "tweet_id.transformer.joint.blue.post_as_float_tensor", + DataType.FLOAT, + List(128L).map(long2Long).asJava, + ) +} + +object UserHistoryTransformerEmbeddingsHomeBlueAdapter extends TransformerEmbeddingsAdapter { + override val embeddingsFeature: Feature.Tensor = + TransformerEmbeddingsFeatures.UserHistoryTransformerEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set.empty +} + +object UserHistoryTransformerEmbeddingsHomeGreenAdapter extends TransformerByteEmbeddingsAdapter { + override val embeddingsFeature: Feature.Tensor = + TransformerEmbeddingsFeatures.UserHistoryTransformerEmbeddingsGreenFeature + + override val commonFeatures: Set[Feature[_]] = Set.empty +} + +object UserHistoryTransformerEmbeddingsJointBlueAdapter extends TransformerByteEmbeddingsAdapter { + override val embeddingsFeature: Feature.Tensor = + TransformerEmbeddingsFeatures.UserHistoryTransformerEmbeddingsJointBlueFeature + + override val commonFeatures: Set[Feature[_]] = Set.empty +} + +object PostTransformerEmbeddingsHomeBlueAdapter extends TransformerEmbeddingsAdapter { + override val embeddingsFeature: Feature.Tensor = + TransformerEmbeddingsFeatures.PostTransformerEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set.empty +} + +object PostTransformerEmbeddingsHomeGreenAdapter extends TransformerEmbeddingsAdapter { + override val embeddingsFeature: Feature.Tensor = + TransformerEmbeddingsFeatures.PostTransformerEmbeddingsGreenFeature + + override val commonFeatures: Set[Feature[_]] = Set.empty +} + +object PostTransformerEmbeddingsJointBlueAdapter extends TransformerEmbeddingsAdapter { + override val embeddingsFeature: Feature.Tensor = + TransformerEmbeddingsFeatures.PostTransformerEmbeddingsJointBlueFeature + + override val commonFeatures: Set[Feature[_]] = Set.empty +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/UserHistoryEventsAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/UserHistoryEventsAdapter.scala new file mode 100644 index 000000000..a3788e67c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/UserHistoryEventsAdapter.scala @@ -0,0 +1,109 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings + +import com.twitter.ml.api.DataType +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.GeneralTensor +import com.twitter.ml.api.Int32Tensor +import com.twitter.ml.api.Int64Tensor +import com.twitter.ml.api.RawTypedTensor +import com.twitter.ml.api.RichDataRecord +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.user_history_transformer.thriftscala.UserHistory +import scala.collection.JavaConverters._ +import java.nio.ByteBuffer + +object UserHistoryEventsFeatures { + val TweetIdsFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_tweet_ids", + DataType.INT64, + List(100L).map(long2Long).asJava, + ) + + val AuthorIdsFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_author_ids", + DataType.INT64, + List(100L).map(long2Long).asJava, + ) + + val ActionTypesFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_action_types", + DataType.INT32, + List(100L).map(long2Long).asJava, + ) + + val ActionTimestampsFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_action_timestamps", + DataType.INT64, + List(100L).map(long2Long).asJava, + ) + + val SemanticIdsFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_semantic_ids", + DataType.UINT8, + List(500L).map(long2Long).asJava, + ) +} + +trait BaseUserHistoryEventsAdapter extends TimelinesMutatingAdapterBase[Seq[UserHistory]] { + override val commonFeatures: Set[Feature[_]] = Set.empty + + private val DefaultSemanticId: Seq[Short] = Seq(0, 0, 0, 0, 0) + + def setAdditionalFeatures( + record: Seq[UserHistory], + mutableDataRecord: RichDataRecord + ): Unit = {} + + override def setFeatures( + record: Seq[UserHistory], + mutableDataRecord: RichDataRecord + ): Unit = { + val tweetIdsFeature = GeneralTensor.int64Tensor( + new Int64Tensor( + record + .map(history => history.sourceTweetId.getOrElse(history.tweetId)).map(long2Long).asJava)) + + val authorIdsFeature = GeneralTensor.int64Tensor( + new Int64Tensor(record.map(_.authorId.getOrElse(-1L)).map(long2Long).asJava)) + + val actionTypesFeature = GeneralTensor.int32Tensor( + new Int32Tensor(record.map(_.actionType.getValue()).map(int2Integer).asJava)) + + val actionTimestampsFeature = GeneralTensor.int64Tensor( + new Int64Tensor(record.map(_.engagedTimestampMs).map(long2Long).asJava)) + + val semanticIdsFeature = GeneralTensor.rawTypedTensor( + new RawTypedTensor( + DataType.UINT8, + ByteBuffer.wrap( + record + .flatMap(_.metadata.flatMap(_.semanticId).getOrElse(DefaultSemanticId)) + .map(_.toByte).toArray) + ) + ) + + mutableDataRecord + .setFeatureValue(UserHistoryEventsFeatures.TweetIdsFeature, tweetIdsFeature) + mutableDataRecord + .setFeatureValue(UserHistoryEventsFeatures.AuthorIdsFeature, authorIdsFeature) + mutableDataRecord + .setFeatureValue(UserHistoryEventsFeatures.ActionTypesFeature, actionTypesFeature) + mutableDataRecord + .setFeatureValue(UserHistoryEventsFeatures.ActionTimestampsFeature, actionTimestampsFeature) + mutableDataRecord + .setFeatureValue(UserHistoryEventsFeatures.SemanticIdsFeature, semanticIdsFeature) + + setAdditionalFeatures(record, mutableDataRecord) + + } + + override def getFeatureContext: FeatureContext = new FeatureContext( + UserHistoryEventsFeatures.TweetIdsFeature, + UserHistoryEventsFeatures.AuthorIdsFeature, + UserHistoryEventsFeatures.ActionTypesFeature, + UserHistoryEventsFeatures.ActionTimestampsFeature, + UserHistoryEventsFeatures.SemanticIdsFeature, + ) +} +object UserHistoryEventsAdapter extends BaseUserHistoryEventsAdapter {} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/VideoUserHistoryEventsAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/VideoUserHistoryEventsAdapter.scala new file mode 100644 index 000000000..1aeab063f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/VideoUserHistoryEventsAdapter.scala @@ -0,0 +1,146 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings + +import com.twitter.ml.api.DataType +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.GeneralTensor +import com.twitter.ml.api.Int32Tensor +import com.twitter.ml.api.Int64Tensor +import com.twitter.ml.api.RichDataRecord +import com.twitter.ml.api.StringTensor +import com.twitter.user_history_transformer.thriftscala.UserHistory +import scala.collection.JavaConverters._ + +object VideoUserHistoryEventsFeatures { + val VideoWatchTimesFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_video_watch_times", + DataType.INT64, + List(100L).map(long2Long).asJava, + ) + val VideoMediaIdsFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_video_media_ids", + DataType.INT64, + List(100L).map(long2Long).asJava, + ) + + val VideoMediaCategoriesFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_video_media_categories", + DataType.INT32, + List(100L).map(long2Long).asJava, + ) + + val VideoDurationsFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_video_durations", + DataType.INT32, + List(100L).map(long2Long).asJava, + ) + val VideoTopEntityFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_video_top_entity_info", + DataType.STRING, + List(100L).map(long2Long).asJava, + ) + val VideoGrokTagsInfoFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_video_grok_tags_info", + DataType.STRING, + List(100L).map(long2Long).asJava, + ) + val VideoCluster95IdsFeature: Feature.Tensor = new Feature.Tensor( + "user.history.user_history_events_video_cluster95_ids_info", + DataType.INT64, + List(100L).map(long2Long).asJava, + ) +} + +object VideoUserHistoryEventsAdapter extends BaseUserHistoryEventsAdapter { + override def getFeatureContext: FeatureContext = { + super.getFeatureContext + .addFeatures(VideoUserHistoryEventsFeatures.VideoWatchTimesFeature) + .addFeatures(VideoUserHistoryEventsFeatures.VideoMediaIdsFeature) + .addFeatures(VideoUserHistoryEventsFeatures.VideoMediaCategoriesFeature) + .addFeatures(VideoUserHistoryEventsFeatures.VideoDurationsFeature) + .addFeatures(VideoUserHistoryEventsFeatures.VideoCluster95IdsFeature) + .addFeatures(VideoUserHistoryEventsFeatures.VideoTopEntityFeature) + .addFeatures(VideoUserHistoryEventsFeatures.VideoGrokTagsInfoFeature) + } + + override def setAdditionalFeatures( + record: Seq[UserHistory], + mutableDataRecord: RichDataRecord + ): Unit = { + val videoWatchTimes = new GeneralTensor() + videoWatchTimes.setInt64Tensor( + new Int64Tensor( + record.flatMap(_.metadata.flatMap(_.watchTime).map(_.toLong)).map(long2Long).asJava) + ) + mutableDataRecord.setFeatureValue( + VideoUserHistoryEventsFeatures.VideoWatchTimesFeature, + videoWatchTimes + ) + val videoMediaIds = new GeneralTensor() + videoMediaIds.setInt64Tensor( + new Int64Tensor(record.flatMap(_.metadata.flatMap(_.mediaId)).map(long2Long).asJava) + ) + mutableDataRecord.setFeatureValue( + VideoUserHistoryEventsFeatures.VideoMediaIdsFeature, + videoMediaIds + ) + + val videoMediaCategories = new GeneralTensor() + videoMediaCategories.setInt32Tensor( + new Int32Tensor( + record.flatMap(_.metadata).flatMap(_.mediaCategory).map(_.value).map(int2Integer).asJava)) + mutableDataRecord.setFeatureValue( + VideoUserHistoryEventsFeatures.VideoMediaCategoriesFeature, + videoMediaCategories + ) + + val videoDurations = new GeneralTensor() + videoDurations.setInt32Tensor( + new Int32Tensor(record.flatMap(_.metadata).flatMap(_.videoDuration).map(int2Integer).asJava)) + mutableDataRecord.setFeatureValue( + VideoUserHistoryEventsFeatures.VideoDurationsFeature, + videoDurations + ) + + val topEntitiesInfo = new GeneralTensor() + val topEntitiesAsString: Seq[String] = record.map { userHistory => + userHistory.metadata + .flatMap(_.entities) + .map(_.filter(_.score.isDefined)) + .filter(_.nonEmpty) + .map(_.maxBy(_.score.get)) + .headOption + .map(entity => entity.qualifiedId._1.toString + ":" + entity.qualifiedId._2.toString) + .getOrElse("") + } + + topEntitiesInfo.setStringTensor(new StringTensor(topEntitiesAsString.asJava)) + mutableDataRecord.setFeatureValue( + VideoUserHistoryEventsFeatures.VideoTopEntityFeature, + topEntitiesInfo + ) + + val grokTagsInfo = new GeneralTensor() + val grokTagsAsString: Seq[String] = record.map { userHistory => + userHistory.metadata + .flatMap(_.tags) + .map(_.map(_.tag)) + .getOrElse(Nil) + .mkString(":") + } + grokTagsInfo.setStringTensor(new StringTensor(grokTagsAsString.asJava)) + + mutableDataRecord.setFeatureValue( + VideoUserHistoryEventsFeatures.VideoGrokTagsInfoFeature, + grokTagsInfo + ) + + val clusterIdsFeature = new GeneralTensor() + clusterIdsFeature.setInt64Tensor( + new Int64Tensor(record.flatMap(_.metadata).flatMap(_.clusterId).map(long2Long).asJava)) + mutableDataRecord.setFeatureValue( + VideoUserHistoryEventsFeatures.VideoCluster95IdsFeature, + clusterIdsFeature + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel new file mode 100644 index 000000000..83515254c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel @@ -0,0 +1,13 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "src/java/com/twitter/ml/api:api-base", + "src/scala/com/twitter/ml/api/util", + "src/scala/com/twitter/timelines/prediction/common/adapters:base", + "src/thrift/com/twitter/ml/api:data-java", + "src/thrift/com/twitter/ml/api:embedding-scala", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala new file mode 100644 index 000000000..a4498d7b2 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala @@ -0,0 +1,152 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings + +import com.twitter.ml.api.DataType +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase + +sealed trait TwhinEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ml.FloatTensor]] { + def twhinEmbeddingsFeature: Feature.Tensor + + override def getFeatureContext: FeatureContext = new FeatureContext( + twhinEmbeddingsFeature + ) + + override def setFeatures( + embedding: Option[ml.FloatTensor], + richDataRecord: RichDataRecord + ): Unit = { + embedding.foreach { floatTensor => + richDataRecord.setFeatureValue( + twhinEmbeddingsFeature, + ScalaToJavaDataRecordConversions.scalaTensor2Java( + ml.GeneralTensor + .FloatTensor(floatTensor))) + } + } +} + +object TwhinEmbeddingsFeatures { + val TwhinAuthorFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "original_author.twhin.tw_hi_n.author_follow_as_float_tensor", + DataType.FLOAT + ) + + val TwhinUserEngagementEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "user.twhin.tw_hi_n.user_engagement_as_float_tensor", + DataType.FLOAT + ) + + val TwhinRebuildUserEngagementEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "user.twhin.tw_hi_n.rebuild_user_engagement_as_float_tensor", + DataType.FLOAT + ) + + val TwhinUserFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "user.twhin.tw_hi_n.user_follow_as_float_tensor", + DataType.FLOAT + ) + + val TwhinUserPositiveEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "user.twhin.tw_hi_n.user_positive_as_float_tensor", + DataType.FLOAT + ) + + val TwhinRebuildUserPositiveEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "user.twhin.tw_hi_n.rebuild_user_positive_as_float_tensor", + DataType.FLOAT + ) + + val TwhinUserNegativeEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "user.twhin.tw_hi_n.user_negative_as_float_tensor", + DataType.FLOAT + ) + + val TwhinTweetEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "original_tweet.twhin.tw_hi_n.tweet_v_2_as_float_tensor", + DataType.FLOAT + ) + + val TwhinRebuildTweetEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "original_tweet.twhin.tw_hi_n.rebuild_tweet_as_float_tensor", + DataType.FLOAT + ) + + val TwhinVideoEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( + "original_tweet.twhin.tw_hi_n.video_as_float_tensor", + DataType.FLOAT + ) +} + +object TwhinAuthorFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter { + override val twhinEmbeddingsFeature: Feature.Tensor = + TwhinEmbeddingsFeatures.TwhinAuthorFollowEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set.empty +} + +object TwhinUserEngagementEmbeddingsAdapter extends TwhinEmbeddingsAdapter { + override val twhinEmbeddingsFeature: Feature.Tensor = + TwhinEmbeddingsFeatures.TwhinUserEngagementEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) +} + +object TwhinRebuildUserEngagementEmbeddingsAdapter extends TwhinEmbeddingsAdapter { + override val twhinEmbeddingsFeature: Feature.Tensor = + TwhinEmbeddingsFeatures.TwhinRebuildUserEngagementEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) +} + +object TwhinUserFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter { + override val twhinEmbeddingsFeature: Feature.Tensor = + TwhinEmbeddingsFeatures.TwhinUserFollowEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) +} + +object TwhinUserPositiveEmbeddingsAdapter extends TwhinEmbeddingsAdapter { + override val twhinEmbeddingsFeature: Feature.Tensor = + TwhinEmbeddingsFeatures.TwhinUserPositiveEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) +} + +object TwhinRebuildUserPositiveEmbeddingsAdapter extends TwhinEmbeddingsAdapter { + override val twhinEmbeddingsFeature: Feature.Tensor = + TwhinEmbeddingsFeatures.TwhinRebuildUserPositiveEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) +} + +object TwhinTweetEmbeddingsAdapter extends TwhinEmbeddingsAdapter { + override val twhinEmbeddingsFeature: Feature.Tensor = + TwhinEmbeddingsFeatures.TwhinTweetEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set.empty +} + +object TwhinRebuildTweetEmbeddingsAdapter extends TwhinEmbeddingsAdapter { + override val twhinEmbeddingsFeature: Feature.Tensor = + TwhinEmbeddingsFeatures.TwhinRebuildTweetEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set.empty +} + +object TwhinVideoEmbeddingsAdapter extends TwhinEmbeddingsAdapter { + override val twhinEmbeddingsFeature: Feature.Tensor = + TwhinEmbeddingsFeatures.TwhinVideoEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set.empty +} + +object TwhinUserNegativeEmbeddingsAdapter extends TwhinEmbeddingsAdapter { + override val twhinEmbeddingsFeature: Feature.Tensor = + TwhinEmbeddingsFeatures.TwhinUserNegativeEmbeddingsFeature + + override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala new file mode 100644 index 000000000..913a0b561 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala @@ -0,0 +1,38 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates + +import com.twitter.ml.api.FeatureContext +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateRootFeature +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.TypedAggregateGroup +import scala.jdk.CollectionConverters.asJavaIterableConverter + +// A helper class deriving aggregate feature info from the given configuration parameters. +class AggregateFeatureInfo( + val aggregateGroups: Set[AggregateGroup], + val aggregateType: AggregateType) { + + private val typedAggregateGroups = aggregateGroups.flatMap(_.buildTypedAggregateGroups()).toList + + val featureContext: FeatureContext = + new FeatureContext( + (typedAggregateGroups.flatMap(_.allOutputFeatures) ++ + typedAggregateGroups.flatMap(_.allOutputKeys) ++ + Seq(TypedAggregateGroup.timestampFeature)).asJava) + + val feature: BaseAggregateRootFeature = + AggregateFeatureInfo.pickFeature(aggregateType) +} + +object AggregateFeatureInfo { + val features: Set[BaseAggregateRootFeature] = + Set(PartAAggregateRootFeature, PartBAggregateRootFeature) + + def pickFeature(aggregateType: AggregateType): BaseAggregateRootFeature = { + val filtered = features.filter(_.aggregateTypes.contains(aggregateType)) + require( + filtered.size == 1, + "requested AggregateType must be backed by exactly one physical store.") + filtered.head + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BUILD.bazel new file mode 100644 index 000000000..ce56d974b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BUILD.bazel @@ -0,0 +1,24 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/offline_aggregates", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "servo/repo/src/main/scala", + "src/java/com/twitter/ml/api:api-base", + "src/scala/com/twitter/timelines/prediction/adapters/request_context", + "src/scala/com/twitter/timelines/prediction/common/adapters:base", + "src/scala/com/twitter/timelines/prediction/common/aggregates", + "src/thrift/com/twitter/user_session_store:thrift-java", + "timelines/data_processing/jobs/timeline_ranking_user_features:mini", + "timelines/data_processing/ml_util/aggregation_framework:common_types", + "timelines/data_processing/ml_util/aggregation_framework/conversion:for-timelines", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..0cf770fb8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala @@ -0,0 +1,124 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates + +import com.twitter.home_mixer.functional_component.feature_hydrator.WithDefaultFeatureMap +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.IRecordOneToOneAdapter +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.AggregateFeaturesToDecodeWithMetadata +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateRootFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType +import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord +import com.twitter.util.Future +import java.lang.{Long => JLong} +import java.util.{Map => JMap} + +abstract case class BaseEdgeAggregateFeature( + name: String, + aggregateGroups: Set[AggregateGroup], + aggregateType: AggregateType, + extractMapFn: AggregateFeaturesToDecodeWithMetadata => JMap[JLong, DenseCompactDataRecord], + adapter: IRecordOneToOneAdapter[Seq[DataRecord]], + getSecondaryKeysFn: CandidateWithFeatures[TweetCandidate] => Seq[Long]) + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord + + private val rootFeatureInfo = new AggregateFeatureInfo(aggregateGroups, aggregateType) + val featureContext: FeatureContext = rootFeatureInfo.featureContext + val rootFeature: BaseAggregateRootFeature = rootFeatureInfo.feature + + override def hashCode(): Int = name.hashCode +} + +trait BaseEdgeAggregateFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with WithDefaultFeatureMap { + + def aggregateFeatures: Set[BaseEdgeAggregateFeature] + + override def features = aggregateFeatures.asInstanceOf[Set[Feature[_, _]]] + + private val batchSize = 32 + + override lazy val defaultFeatureMap: FeatureMap = { + val featureMapBuilder = new FeatureMapBuilder() + aggregateFeatures.foreach(feature => featureMapBuilder.add(feature, feature.defaultValue)) + featureMapBuilder.build() + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val featuresSeq = aggregateFeatures.toSeq + val dataRecordsFuture = featuresSeq.map { feature => + hydrateAggregateFeature(query, candidates, feature) + } + + Future.collect(dataRecordsFuture).map { dataRecordsPerFeature => + val dataRecordsPerCandidate = dataRecordsPerFeature.transpose + dataRecordsPerCandidate.map { dataRecords => + assert(featuresSeq.size == dataRecords.size) + val builder = FeatureMapBuilder(sizeHint = featuresSeq.size) + featuresSeq.zip(dataRecords).map { + case (feature, dataRecord) => builder.add(feature, dataRecord) + } + builder.build() + } + } + } + + private def getDataRecord( + ids: Seq[Long], + decoded: Map[Long, DataRecord], + feature: BaseEdgeAggregateFeature + ) = { + val dataRecords = ids.flatMap(decoded.get) + feature.adapter.adaptToDataRecord(dataRecords) + } + + private def hydrateAggregateFeature( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + feature: BaseEdgeAggregateFeature + ): Future[Seq[DataRecord]] = { + val rootFeature = feature.rootFeature + val extractMapFn = feature.extractMapFn + val featureContext = feature.featureContext + val secondaryIds: Seq[Seq[Long]] = candidates.map(feature.getSecondaryKeysFn) + + val featuresToDecodeWithMetadata = query.features + .flatMap(_.getOrElse(rootFeature, None)) + .getOrElse(AggregateFeaturesToDecodeWithMetadata.empty) + + // Decode the DenseCompactDataRecords into DataRecords for each required secondary id. + val decoded: Map[Long, DataRecord] = Utils.selectAndTransform( + secondaryIds.flatten.distinct, + featuresToDecodeWithMetadata.toDataRecord, + extractMapFn(featuresToDecodeWithMetadata) + ) + + // Remove unnecessary features in-place. This is safe because the underlying DataRecords + // are unique and have just been generated in the previous step. + decoded.values.foreach(Utils.filterDataRecord(_, featureContext)) + + // Put features into the FeatureMapBuilders + + OffloadFuturePools.offloadBatchElementToElement( + secondaryIds, + getDataRecord(_, decoded, feature), + batchSize) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala new file mode 100644 index 000000000..01984a51d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala @@ -0,0 +1,127 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates + +import com.twitter.home_mixer.functional_component.feature_hydrator.TSPInferredTopicFeature +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates.PassThroughAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates.SparseAggregatesToDenseAdapter +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature +import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType +import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig +import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig.CombineCountPolicies + +object EdgeAggregateFeatures { + + object UserAuthorAggregateFeature + extends BaseEdgeAggregateFeature( + name = "UserAuthorAggregateFeature", + aggregateGroups = TimelinesAggregationConfig.userAuthorAggregatesV2 ++ Set( + TimelinesAggregationConfig.userAuthorAggregatesV5, + TimelinesAggregationConfig.tweetSourceUserAuthorAggregatesV1, + TimelinesAggregationConfig.twitterWideUserAuthorAggregates + ), + aggregateType = AggregateType.UserAuthor, + extractMapFn = _.userAuthorAggregates, + adapter = PassThroughAdapter, + getSecondaryKeysFn = _.features.getOrElse(AuthorIdFeature, None).toSeq + ) + + object UserOriginalAuthorAggregateFeature + extends BaseEdgeAggregateFeature( + name = "UserOriginalAuthorAggregateFeature", + aggregateGroups = Set(TimelinesAggregationConfig.userOriginalAuthorAggregatesV1), + aggregateType = AggregateType.UserOriginalAuthor, + extractMapFn = _.userOriginalAuthorAggregates, + adapter = PassThroughAdapter, + getSecondaryKeysFn = candidate => + CandidatesUtil.getOriginalAuthorId(candidate.features).toSeq + ) + + object UserTopicAggregateFeature + extends BaseEdgeAggregateFeature( + name = "UserTopicAggregateFeature", + aggregateGroups = Set( + TimelinesAggregationConfig.userTopicAggregates, + TimelinesAggregationConfig.userTopicAggregatesV2, + ), + aggregateType = AggregateType.UserTopic, + extractMapFn = _.userTopicAggregates, + adapter = PassThroughAdapter, + getSecondaryKeysFn = candidate => + candidate.features.getOrElse(TopicIdSocialContextFeature, None).toSeq + ) + + object UserMentionAggregateFeature + extends BaseEdgeAggregateFeature( + name = "UserMentionAggregateFeature", + aggregateGroups = Set(TimelinesAggregationConfig.userMentionAggregates), + aggregateType = AggregateType.UserMention, + extractMapFn = _.userMentionAggregates, + adapter = new SparseAggregatesToDenseAdapter(CombineCountPolicies.MentionCountsPolicy), + getSecondaryKeysFn = candidate => + candidate.features.getOrElse(MentionScreenNameFeature, Seq.empty).map(_.hashCode.toLong) + ) + + object UserInferredTopicAggregateFeature + extends BaseEdgeAggregateFeature( + name = "UserInferredTopicAggregateFeature", + aggregateGroups = Set( + TimelinesAggregationConfig.userInferredTopicAggregates, + ), + aggregateType = AggregateType.UserInferredTopic, + extractMapFn = _.userInferredTopicAggregates, + adapter = new SparseAggregatesToDenseAdapter( + CombineCountPolicies.UserInferredTopicCountsPolicy), + getSecondaryKeysFn = candidate => + candidate.features.getOrElse(TSPInferredTopicFeature, Map.empty[Long, Double]).keys.toSeq + ) + + object UserInferredTopicAggregateV2Feature + extends BaseEdgeAggregateFeature( + name = "UserInferredTopicAggregateV2Feature", + aggregateGroups = Set( + TimelinesAggregationConfig.userInferredTopicAggregatesV2 + ), + aggregateType = AggregateType.UserInferredTopic, + extractMapFn = _.userInferredTopicAggregates, + adapter = new SparseAggregatesToDenseAdapter( + CombineCountPolicies.UserInferredTopicV2CountsPolicy), + getSecondaryKeysFn = candidate => + candidate.features.getOrElse(TSPInferredTopicFeature, Map.empty[Long, Double]).keys.toSeq + ) + + object UserMediaUnderstandingAnnotationAggregateFeature + extends BaseEdgeAggregateFeature( + name = "UserMediaUnderstandingAnnotationAggregateFeature", + aggregateGroups = Set( + TimelinesAggregationConfig.userMediaUnderstandingAnnotationAggregates), + aggregateType = AggregateType.UserMediaUnderstandingAnnotation, + extractMapFn = _.userMediaUnderstandingAnnotationAggregates, + adapter = new SparseAggregatesToDenseAdapter( + CombineCountPolicies.UserMediaUnderstandingAnnotationCountsPolicy), + getSecondaryKeysFn = candidate => + CandidatesUtil.getMediaUnderstandingAnnotationIds(candidate.features) + ) + + object UserEngagerAggregateFeature + extends BaseEdgeAggregateFeature( + name = "UserEngagerAggregateFeature", + aggregateGroups = Set(TimelinesAggregationConfig.userEngagerAggregates), + aggregateType = AggregateType.UserEngager, + extractMapFn = _.userEngagerAggregates, + adapter = new SparseAggregatesToDenseAdapter(CombineCountPolicies.EngagerCountsPolicy), + getSecondaryKeysFn = candidate => CandidatesUtil.getEngagerUserIds(candidate.features) + ) + + object UserEngagerGoodClickAggregateFeature + extends BaseEdgeAggregateFeature( + name = "UserEngagerGoodClickAggregateFeature", + aggregateGroups = Set(TimelinesAggregationConfig.userEngagerGoodClickAggregates), + aggregateType = AggregateType.UserEngager, + extractMapFn = _.userEngagerAggregates, + adapter = new SparseAggregatesToDenseAdapter( + CombineCountPolicies.EngagerGoodClickCountsPolicy), + getSecondaryKeysFn = candidate => CandidatesUtil.getEngagerUserIds(candidate.features) + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala new file mode 100644 index 000000000..72a2c28d4 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala @@ -0,0 +1,37 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates + +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartARepository +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateRootFeature +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.servo.repository.Repository +import com.twitter.timelines.data_processing.jobs.timeline_ranking_user_features.TimelinesPartAStoreRegister +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig +import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata +import com.twitter.user_session_store.thriftjava.UserSession +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +object PartAAggregateRootFeature extends BaseAggregateRootFeature { + override val aggregateStores: Set[StoreConfig[_]] = TimelinesPartAStoreRegister.allStores +} + +@Singleton +class PartAAggregateQueryFeatureHydrator @Inject() ( + @Named(TimelineAggregatePartARepository) + repository: Repository[Long, Option[UserSession]], + @Named(TimelineAggregateMetadataRepository) + metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) + extends BaseAggregateQueryFeatureHydrator( + repository, + metadataRepository, + PartAAggregateRootFeature + ) { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("PartAAggregateQuery") + + override val features = Set(PartAAggregateRootFeature) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala new file mode 100644 index 000000000..2bee82f84 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala @@ -0,0 +1,146 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates + +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartBRepository +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.DataRecordMerger +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateRootFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.repository.Repository +import com.twitter.stitch.Stitch +import com.twitter.timelines.data_processing.jobs.timeline_ranking_user_features.TimelinesPartBStoreRegister +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig +import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter +import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig +import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata +import com.twitter.user_session_store.thriftjava.UserSession +import com.twitter.util.Time +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +object PartBAggregateRootFeature extends BaseAggregateRootFeature { + override val aggregateStores: Set[StoreConfig[_]] = TimelinesPartBStoreRegister.allStores +} + +object UserAggregateFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class PartBAggregateQueryFeatureHydrator @Inject() ( + @Named(TimelineAggregatePartBRepository) + repository: Repository[Long, Option[UserSession]], + @Named(TimelineAggregateMetadataRepository) + metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) + extends BaseAggregateQueryFeatureHydrator( + repository, + metadataRepository, + PartBAggregateRootFeature + ) { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("PartBAggregateQuery") + + override val features: Set[Feature[_, _]] = + Set(PartBAggregateRootFeature, UserAggregateFeature) + + private val userAggregateFeatureInfo = new AggregateFeatureInfo( + aggregateGroups = Set( + TimelinesAggregationConfig.userAggregatesV2, + TimelinesAggregationConfig.userAggregatesV5Continuous, + TimelinesAggregationConfig.userAggregatesV6, + TimelinesAggregationConfig.twitterWideUserAggregates, + ), + aggregateType = AggregateType.User + ) + + private val userHourAggregateFeatureInfo = new AggregateFeatureInfo( + aggregateGroups = Set( + TimelinesAggregationConfig.userRequestHourAggregates, + ), + aggregateType = AggregateType.UserRequestHour + ) + + private val userDowAggregateFeatureInfo = new AggregateFeatureInfo( + aggregateGroups = Set( + TimelinesAggregationConfig.userRequestDowAggregates + ), + aggregateType = AggregateType.UserRequestDow + ) + + require( + userAggregateFeatureInfo.feature == PartBAggregateRootFeature, + "UserAggregates feature must be provided by the PartB data source.") + require( + userHourAggregateFeatureInfo.feature == PartBAggregateRootFeature, + "UserRequstHourAggregates feature must be provided by the PartB data source.") + require( + userDowAggregateFeatureInfo.feature == PartBAggregateRootFeature, + "UserRequestDowAggregates feature must be provided by the PartB data source.") + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + // Hydrate TimelineAggregatePartBFeature and UserAggregateFeature sequentially. + super.hydrate(query).map { featureMap => + val time: Time = Time.now + val hourOfDay = RequestContextAdapter.hourFromTimestamp(time.inMilliseconds) + val dayOfWeek = RequestContextAdapter.dowFromTimestamp(time.inMilliseconds) + + val dr = featureMap + .get(PartBAggregateRootFeature).map { featuresWithMetadata => + val userAggregatesDr = + featuresWithMetadata.userAggregatesOpt + .map(featuresWithMetadata.toDataRecord) + val userRequestHourAggregatesDr = + Option(featuresWithMetadata.userRequestHourAggregates.get(hourOfDay)) + .map(featuresWithMetadata.toDataRecord) + val userRequestDowAggregatesDr = + Option(featuresWithMetadata.userRequestDowAggregates.get(dayOfWeek)) + .map(featuresWithMetadata.toDataRecord) + + dropUnknownFeatures(userAggregatesDr, userAggregateFeatureInfo.featureContext) + + dropUnknownFeatures( + userRequestHourAggregatesDr, + userHourAggregateFeatureInfo.featureContext) + + dropUnknownFeatures( + userRequestDowAggregatesDr, + userDowAggregateFeatureInfo.featureContext) + + mergeDataRecordOpts( + userAggregatesDr, + userRequestHourAggregatesDr, + userRequestDowAggregatesDr) + + }.getOrElse(new DataRecord()) + + featureMap + (UserAggregateFeature, dr) + } + } + + private val drMerger = new DataRecordMerger + private def mergeDataRecordOpts(dataRecordOpts: Option[DataRecord]*): DataRecord = + dataRecordOpts.flatten.foldLeft(new DataRecord) { (l, r) => + drMerger.merge(l, r) + l + } + + private def dropUnknownFeatures( + dataRecordOpt: Option[DataRecord], + featureContext: FeatureContext + ): Unit = + dataRecordOpt.foreach(new RichDataRecord(_, featureContext).dropUnknownFeatures()) + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TopicEdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TopicEdgeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..26415a531 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TopicEdgeAggregateFeatureHydrator.scala @@ -0,0 +1,26 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates + +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateFeature +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateV2Feature +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserTopicAggregateFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTopicEdgeAggregateFeatureHydratorParam +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object TopicEdgeAggregateFeatureHydrator + extends BaseEdgeAggregateFeatureHydrator + with Conditionally[PipelineQuery] { + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableTopicEdgeAggregateFeatureHydratorParam) + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TopicEdgeAggregate") + + override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( + UserInferredTopicAggregateFeature, + UserInferredTopicAggregateV2Feature, + UserTopicAggregateFeature, + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TopicEdgeTruncatedAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TopicEdgeTruncatedAggregateFeatureHydrator.scala new file mode 100644 index 000000000..a8d54322e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TopicEdgeTruncatedAggregateFeatureHydrator.scala @@ -0,0 +1,24 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates + +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateFeature +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateV2Feature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTopicEdgeAggregateFeatureHydratorParam +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object TopicEdgeTruncatedAggregateFeatureHydrator + extends BaseEdgeAggregateFeatureHydrator + with Conditionally[PipelineQuery] { + + override def onlyIf(query: PipelineQuery): Boolean = + !query.params(EnableTopicEdgeAggregateFeatureHydratorParam) + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TopicEdgeTruncatedAggregate") + + override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( + UserInferredTopicAggregateFeature, + UserInferredTopicAggregateV2Feature, + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TweetContentEdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TweetContentEdgeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..4df52cd30 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TweetContentEdgeAggregateFeatureHydrator.scala @@ -0,0 +1,14 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates + +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserMediaUnderstandingAnnotationAggregateFeature +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier + +object TweetContentEdgeAggregateFeatureHydrator extends BaseEdgeAggregateFeatureHydrator { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TweetContentEdgeAggregate") + + override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( + UserMediaUnderstandingAnnotationAggregateFeature + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/UserEngagerEdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/UserEngagerEdgeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..240b00df1 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/UserEngagerEdgeAggregateFeatureHydrator.scala @@ -0,0 +1,16 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates + +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerAggregateFeature +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerGoodClickAggregateFeature +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier + +object UserEngagerEdgeAggregateFeatureHydrator extends BaseEdgeAggregateFeatureHydrator { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserEngagerEdgeAggregate") + + override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( + UserEngagerAggregateFeature, + UserEngagerGoodClickAggregateFeature, + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/UserEntityEdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/UserEntityEdgeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..a8e2f6dba --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/UserEntityEdgeAggregateFeatureHydrator.scala @@ -0,0 +1,18 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates + +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserAuthorAggregateFeature +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserMentionAggregateFeature +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserOriginalAuthorAggregateFeature +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier + +object UserEntityAggregateFeatureHydrator extends BaseEdgeAggregateFeatureHydrator { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserEntityEdgeAggregate") + + override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( + UserAuthorAggregateFeature, + UserOriginalAuthorAggregateFeature, + UserMentionAggregateFeature + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Utils.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Utils.scala new file mode 100644 index 000000000..45d205888 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Utils.scala @@ -0,0 +1,36 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates + +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord + +private[offline_aggregates] object Utils { + + /** + * Selects only those values in map that correspond to the keys in ids and apply the provided + * transform to the selected values. This is a convenience method for use by Timelines Aggregation + * Framework based features. + * + * @param idsToSelect The set of ids to extract values for. + * @param transform A transform to apply to the selected values. + * @param map Map[Long, DenseCompactDataRecord] + */ + def selectAndTransform( + idsToSelect: Seq[Long], + transform: DenseCompactDataRecord => DataRecord, + map: java.util.Map[java.lang.Long, DenseCompactDataRecord], + ): Map[Long, DataRecord] = { + val filtered: Seq[(Long, DataRecord)] = + for { + id <- idsToSelect if map.containsKey(id) + } yield { + id -> transform(map.get(id)) + } + filtered.toMap + } + + def filterDataRecord(dr: DataRecord, featureContext: FeatureContext): Unit = { + new RichDataRecord(dr, featureContext).dropUnknownFeatures() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BUILD.bazel new file mode 100644 index 000000000..7a6fa919a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BUILD.bazel @@ -0,0 +1,35 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/src/jvm/com/twitter/storehaus:core", + "finatra/inject/inject-core/src/main/scala", + "home-mixer-features/thrift/src/main/thrift:thrift-java", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/real_time_aggregates", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/real_time_aggregates", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "servo/repo/src/main/scala", + "src/java/com/twitter/ml/api:api-base", + "src/scala/com/twitter/ml/api", + "src/scala/com/twitter/ml/api:api-base", + "src/scala/com/twitter/storehaus_internal/store", + "src/scala/com/twitter/timelines/prediction/common/aggregates/real_time:base-config", + "src/scala/com/twitter/timelines/prediction/common/aggregates/real_time/tv:base-config", + "src/thrift/com/twitter/timelines/realtime_aggregates:thrift-scala", + ], + exports = [ + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/real_time_aggregates", + "timelines/data_processing/ml_util/aggregation_framework/real_time_aggregates", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..b6c8ae98a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala @@ -0,0 +1,60 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates + +import com.google.inject.name.Named +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.authorIdFeature +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1 +import com.twitter.home_mixer.param.HomeMixerInjectionNames.EngagementsReceivedByAuthorCache +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer_features.{thriftjava => t} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.cache.ReadCache +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup +import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ +import javax.inject.Inject +import javax.inject.Singleton + +object EngagementsReceivedByAuthorRealTimeAggregateFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator @Inject() ( + override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, + @Named(EngagementsReceivedByAuthorCache) override val client: ReadCache[Long, DataRecord], + override val statsReceiver: StatsReceiver) + extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[Long] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("EngagementsReceivedByAuthorRealTimeAggregate") + + override val outputFeature: DataRecordInAFeature[TweetCandidate] = + EngagementsReceivedByAuthorRealTimeAggregateFeature + + override val aggregateGroups: Seq[AggregateGroup] = Seq( + authorEngagementRealTimeAggregatesProd, + authorShareEngagementsRealTimeAggregates + ) + + override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( + authorShareEngagementsRealTimeAggregates -> "original_author.timelines.author_share_engagements_real_time_aggregates." + ) + + def serializeKey(key: Long): String = { + keyTransformD1(authorIdFeature)(key) + } + + override def keysFromQueryAndCandidates( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[Option[Long]] = + candidates.map(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator.scala new file mode 100644 index 000000000..1fc357427 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator.scala @@ -0,0 +1,118 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates + +import com.twitter.home_mixer.functional_component.feature_hydrator.WithDefaultFeatureMap +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableHomeMixerFeaturesService +import com.twitter.home_mixer.util.MissingKeyException +import com.twitter.home_mixer_features.thriftjava.HomeMixerFeaturesRequest +import com.twitter.home_mixer_features.{thriftjava => t} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.real_time_aggregates.BaseRealTimeAggregateBulkCandidateFeatureHydrator +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.util.Future +import com.twitter.util.Return +import com.twitter.util.Throw +import com.twitter.util.Try +import scala.jdk.CollectionConverters.iterableAsScalaIterableConverter +import scala.jdk.CollectionConverters.seqAsJavaListConverter + +trait FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[K] + extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[K, TweetCandidate] + with WithDefaultFeatureMap { + + def EmptyDataRecord: DataRecord = new DataRecord() + + override lazy val defaultFeatureMap: FeatureMap = FeatureMap(outputFeature, EmptyDataRecord) + + def serializeKey(key: K): String + + val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient + + private val batchSize = 64 + + private def createHomeMixerFeaturesRequest(keys: Seq[K]): t.HomeMixerFeaturesRequest = { + val keysSerialized = keys.map(serializeKey) + val request = new HomeMixerFeaturesRequest() + request.setKeys(keysSerialized.asJava) + request.setCache(t.Cache.RTA) + } + + private def unmarshallHomeMixerFeaturesResponse( + response: t.HomeMixerFeaturesResponse + ): Iterable[DataRecord] = { + response.getHomeMixerFeatures.asScala + .map { homeMixerFeatureOpt => + if (homeMixerFeatureOpt.isSetHomeMixerFeaturesType) { + val homeMixerFeature = homeMixerFeatureOpt.getHomeMixerFeaturesType + if (homeMixerFeature.isSet(t.HomeMixerFeaturesType._Fields.DATA_RECORD)) { + postTransformer(homeMixerFeature.getDataRecord) + } else { + throw new Exception("Unexpected type") + } + } else EmptyDataRecord + } + } + + private def getFeatureMaps( + possiblyKeys: Seq[Option[K]], + dataRecordMap: Map[K, DataRecord] + ): Future[Seq[Try[DataRecord]]] = { + val transformer = { key: Option[K] => + if (key.nonEmpty) + Return(dataRecordMap.getOrElse(key.get, EmptyDataRecord)) + else Throw(MissingKeyException) + } + OffloadFuturePools.offloadBatchElementToElement(possiblyKeys, transformer, batchSize) + } + + def fetchRecordsFromHomeMixerFeaturesService( + possiblyKeys: Seq[Option[K]] + ): Future[Seq[Try[DataRecord]]] = { + val keys = possiblyKeys.flatten.distinct + + val transformer = { keyGroup: Seq[K] => + val request = createHomeMixerFeaturesRequest(keyGroup) + val responseFut = + homeMixerFeatureService.getHomeMixerFeatures(request) + responseFut + .map { response => + keyGroup.zip(unmarshallHomeMixerFeaturesResponse(response)) + }.handle { case _ => Seq.empty } + } + + val response = + OffloadFuturePools.offloadBatchSeqToFutureSeq(keys, transformer, batchSize) + + response.map(_.toMap).flatMap { keyValueResult => + getFeatureMaps(possiblyKeys, keyValueResult) + } + } + + def fetchRecords( + possiblyKeys: Seq[Option[K]], + callMiddleMan: Boolean + ): Future[Seq[Try[DataRecord]]] = { + if (callMiddleMan) fetchRecordsFromHomeMixerFeaturesService(possiblyKeys) + else fetchAndConstructDataRecords(possiblyKeys) // cache is default + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val possiblyKeys = keysFromQueryAndCandidates(query, candidates) + val callMiddleMan = query.params(EnableHomeMixerFeaturesService) + fetchRecords(possiblyKeys, callMiddleMan).map { dataRecords => + val featureMaps = dataRecords.map { dataRecord => + FeatureMapBuilder().add(outputFeature, dataRecord).build() + } + featureMaps + } + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..5cc2b2585 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala @@ -0,0 +1,79 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates + +import com.google.inject.name.Named +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.countryCodeFeature +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1T1 +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.topicIdFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTopicCountryBasedRealTimeAggregateFeatureHydratorParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicCountryEngagementCache +import com.twitter.home_mixer_features.{thriftjava => t} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.cache.ReadCache +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup +import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ +import javax.inject.Inject +import javax.inject.Singleton + +object TopicCountryEngagementRealTimeAggregateFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TopicCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( + override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, + @Named(TopicCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord], + override val statsReceiver: StatsReceiver) + extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TopicCountryEngagementRealTimeAggregate") + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableTopicCountryBasedRealTimeAggregateFeatureHydratorParam) + + override val outputFeature: DataRecordInAFeature[TweetCandidate] = + TopicCountryEngagementRealTimeAggregateFeature + + override val aggregateGroups: Seq[AggregateGroup] = Seq( + topicCountryRealTimeAggregates + ) + + override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( + topicCountryRealTimeAggregates -> "topic-country_code.timelines.topic_country_engagement_real_time_aggregates." + ) + + def serializeKey(key: (Long, String)): String = { + keyTransformD1T1(topicIdFeature, countryCodeFeature)(key) + } + + override def keysFromQueryAndCandidates( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[Option[(Long, String)]] = { + candidates.map { candidate => + val maybeTopicId = candidate.features + .getTry(TopicIdSocialContextFeature) + .toOption + .flatten + + val maybeCountryCode = query.clientContext.countryCode + + for { + topicId <- maybeTopicId + countryCode <- maybeCountryCode + } yield (topicId, countryCode) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..848b62311 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala @@ -0,0 +1,74 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1 +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.topicIdFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTopicBasedRealTimeAggregateFeatureHydratorParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicEngagementCache +import com.twitter.home_mixer_features.{thriftjava => t} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.cache.ReadCache +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup +import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +object TopicEngagementRealTimeAggregateFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TopicEngagementRealTimeAggregateFeatureHydrator @Inject() ( + override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, + @Named(TopicEngagementCache) override val client: ReadCache[Long, DataRecord], + override val statsReceiver: StatsReceiver) + extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[Long] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TopicEngagementRealTimeAggregate") + + override val outputFeature: DataRecordInAFeature[TweetCandidate] = + TopicEngagementRealTimeAggregateFeature + + override val aggregateGroups: Seq[AggregateGroup] = Seq( + topicEngagementRealTimeAggregatesProd, + topicEngagement24HourRealTimeAggregatesProd, + topicShareEngagementsRealTimeAggregates + ) + + override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( + topicEngagement24HourRealTimeAggregatesProd -> "topic.timelines.topic_engagement_24_hour_real_time_aggregates.", + topicShareEngagementsRealTimeAggregates -> "topic.timelines.topic_share_engagements_real_time_aggregates." + ) + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableTopicBasedRealTimeAggregateFeatureHydratorParam) + + def serializeKey(key: Long): String = { + keyTransformD1(topicIdFeature)(key) + } + + override def keysFromQueryAndCandidates( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[Option[Long]] = { + candidates.map { candidate => + candidate.features + .getTry(TopicIdSocialContextFeature) + .toOption + .flatten + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..c501d1043 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala @@ -0,0 +1,144 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates + +import com.google.inject.name.Named +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.countryCodeFeature +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1T1 +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.tweetIdFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetCountryRTAMhFallbackParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetCountryRTAMhOnlyParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanStore +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetCountryEngagementCache +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer_features.{thriftjava => t} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ReadCache +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKey +import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ +import com.twitter.util.Future +import com.twitter.util.Try +import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} +import javax.inject.Inject +import javax.inject.Singleton +import com.twitter.stitch.Stitch + +object TweetCountryEngagementRealTimeAggregateFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TweetCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( + override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, + @Named(TweetCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord], + @Named(RTAManhattanStore) mhClient: Option[ReadableStore[thrift.AggregationKey, DataRecord]], + override val statsReceiver: StatsReceiver) + extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TweetCountryEngagementRealTimeAggregate") + + override val outputFeature: DataRecordInAFeature[TweetCandidate] = + TweetCountryEngagementRealTimeAggregateFeature + + override val aggregateGroups: Seq[AggregateGroup] = Seq( + tweetCountryRealTimeAggregates, + tweetCountryPrivateEngagementsRealTimeAggregates + ) + + override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( + tweetCountryRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_engagement_real_time_aggregates.", + tweetCountryPrivateEngagementsRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_private_engagement_real_time_aggregates." + ) + + def serializeKey(key: (Long, String)): String = { + keyTransformD1T1(tweetIdFeature, countryCodeFeature)(key) + } + + override def keysFromQueryAndCandidates( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[Option[(Long, String)]] = { + val countryCode = query.clientContext.countryCode + candidates.map { candidate => + val originalTweetId = CandidatesUtil.getOriginalTweetId(candidate) + countryCode.map((originalTweetId, _)) + } + } + + def convert(key: (Long, String)): thrift.AggregationKey = { + val ak = AggregationKey(Map(tweetIdFeature -> key._1), Map(countryCodeFeature -> key._2)) + thrift.AggregationKey( + ak.discreteFeaturesById, + ak.textFeaturesById + ) + } + + def fetchAndConstructDataRecordFromMh( + possiblyKeys: Seq[Option[(Long, String)]] + ): Future[Seq[Try[DataRecord]]] = { + Future + .collect { + possiblyKeys.flatten + .map { + convert(_) + } + .grouped(64).map { keyGroup => + val results = mhClient.get.multiGet(keyGroup.toSet) + Future.collect(keyGroup.flatMap(results.get)).map { drSeq => + drSeq.map { drOpt => + if (drOpt.isEmpty) statsReceiver.scope("mhTweetCountryRTA").counter("empty").incr() + else statsReceiver.scope("mhTweetCountryRTA").counter("non_empty").incr() + Try(drOpt.map(postTransformer).getOrElse(EmptyDataRecord)) + } + } + }.toSeq + }.map(_.flatten) + } + + def getFetchFunc( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Future[Seq[Try[DataRecord]]] = { + val fetchFromMhOnly = query.params.getBoolean(EnableTweetCountryRTAMhOnlyParam) + val fetchFromMhAsFallBack = query.params.getBoolean(EnableTweetCountryRTAMhFallbackParam) + val possiblyKeys = keysFromQueryAndCandidates(query, candidates) + val stats = statsReceiver.scope("tweet_country_real_time_rta") + if (fetchFromMhOnly) { + fetchAndConstructDataRecordFromMh(possiblyKeys) + } else if (fetchFromMhAsFallBack) { + fetchAndConstructDataRecordsWithFallback( + possiblyKeys, + stats, + fetchAndConstructDataRecords, + fetchAndConstructDataRecordFromMh, + ) + } else { + fetchAndConstructDataRecords(possiblyKeys) + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + getFetchFunc(query, candidates).map { dataRecords => + val featureMaps = dataRecords.map { dataRecord => + FeatureMapBuilder().add(outputFeature, dataRecord).build() + } + featureMaps + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..6dcd78af0 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala @@ -0,0 +1,147 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates + +import com.google.inject.name.Named +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1 +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.tweetIdFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetRTAMhFallbackParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetRTAMhOnlyParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanStore +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetEngagementCache +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer_features.{thriftjava => t} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ReadCache +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKey +import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ +import com.twitter.util.Future +import com.twitter.util.Try +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +object TweetEngagementRealTimeAggregateFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TweetEngagementRealTimeAggregateFeatureHydrator @Inject() ( + override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, + @Named(TweetEngagementCache) override val client: ReadCache[Long, DataRecord], + @Named(RTAManhattanStore) mhClient: Option[ReadableStore[thrift.AggregationKey, DataRecord]], + override val statsReceiver: StatsReceiver) + extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[Long] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TweetEngagementRealTimeAggregate") + + override val outputFeature: DataRecordInAFeature[TweetCandidate] = + TweetEngagementRealTimeAggregateFeature + + val batchSize = 64 + + override val aggregateGroups: Seq[AggregateGroup] = Seq( + tweetEngagement30MinuteCountsProd, + tweetEngagementTotalCountsProd, + tweetEngagementUserStateRealTimeAggregatesProd, + tweetNegativeEngagementUserStateRealTimeAggregates, + tweetNegativeEngagement6HourCounts, + tweetNegativeEngagementTotalCounts, + tweetShareEngagementsRealTimeAggregates, + tweetBCEDwellEngagementsRealTimeAggregates + ) + + override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( + tweetShareEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_share_engagements_real_time_aggregates.", + tweetBCEDwellEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_bce_dwell_engagements_real_time_aggregates." + ) + + def serializeKey(key: Long): String = { + keyTransformD1(tweetIdFeature)(key) + } + + override def keysFromQueryAndCandidates( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[Option[Long]] = { + val keys = candidates + .map(candidate => Some(CandidatesUtil.getOriginalTweetId(candidate))) + keys + } + + def convert(key: Long): thrift.AggregationKey = { + val ak = AggregationKey(Map(tweetIdFeature -> key), Map.empty) + thrift.AggregationKey( + ak.discreteFeaturesById, + ak.textFeaturesById + ) + } + + def fetchAndConstructDataRecordFromMh( + possiblyKeys: Seq[Option[Long]] + ): Future[Seq[Try[DataRecord]]] = { + Future + .collect { + possiblyKeys.flatten + .map { convert(_) } + .grouped(batchSize).map { keyGroup => + val results = mhClient.get.multiGet(keyGroup.toSet) + Future.collect(keyGroup.flatMap(results.get)).map { drSeq => + drSeq.map { drOpt => + if (drOpt.isEmpty) statsReceiver.scope("mhTweetRTA").counter("empty").incr() + else statsReceiver.scope("mhTweetRTA").counter("non_empty").incr() + Try(drOpt.map(postTransformer).getOrElse(EmptyDataRecord)) + } + } + }.toSeq + }.map(_.flatten) + } + + def getFetchFunc( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Future[Seq[Try[DataRecord]]] = { + val fetchFromMhOnly = query.params.getBoolean(EnableTweetRTAMhOnlyParam) + val fetchFromMhAsFallBack = query.params.getBoolean(EnableTweetRTAMhFallbackParam) + val possiblyKeys = keysFromQueryAndCandidates(query, candidates) + val stats = statsReceiver.scope("tweet_real_time_rta") + if (fetchFromMhOnly) { + fetchAndConstructDataRecordFromMh(possiblyKeys) + } else if (fetchFromMhAsFallBack) { + fetchAndConstructDataRecordsWithFallback( + possiblyKeys, + stats, + fetchAndConstructDataRecords, + fetchAndConstructDataRecordFromMh, + ) + } else { + fetchAndConstructDataRecords(possiblyKeys) + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + getFetchFunc(query, candidates).map { dataRecords => + val featureMaps = dataRecords.map { dataRecord => + FeatureMapBuilder().add(outputFeature, dataRecord).build() + } + featureMaps + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..e97ce8db8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala @@ -0,0 +1,59 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates + +import com.google.inject.name.Named +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.ListIdFeature +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1 +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.listIdFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwitterListEngagementCache +import com.twitter.home_mixer_features.{thriftjava => t} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.cache.ReadCache +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup +import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ +import javax.inject.Inject +import javax.inject.Singleton + +object TwitterListEngagementRealTimeAggregateFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TwitterListEngagementRealTimeAggregateFeatureHydrator @Inject() ( + override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, + @Named(TwitterListEngagementCache) override val client: ReadCache[Long, DataRecord], + override val statsReceiver: StatsReceiver) + extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[Long] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TwitterListEngagementRealTimeAggregate") + + override val outputFeature: DataRecordInAFeature[TweetCandidate] = + TwitterListEngagementRealTimeAggregateFeature + + override val aggregateGroups: Seq[AggregateGroup] = + Seq(listEngagementRealTimeAggregatesProd) + + override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( + listEngagementRealTimeAggregatesProd -> "twitter_list.timelines.twitter_list_engagement_real_time_aggregates." + ) + + def serializeKey(key: Long): String = { + keyTransformD1(listIdFeature)(key) + } + + override def keysFromQueryAndCandidates( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[Option[Long]] = candidates.map { candidate => + candidate.features.getTry(ListIdFeature).toOption.flatten + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..27ca45b2f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala @@ -0,0 +1,147 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates + +import com.google.inject.name.Named +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.authorIdFeature +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD2 +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD2AggregationKey +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.userIdFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableUserAuthorRTAMhFallbackParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnableUserAuthorRTAMhOnlyParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanStore +import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserAuthorEngagementCache +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer_features.{thriftjava => t} +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ReadCache +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup +import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ +import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} +import com.twitter.util.Future +import com.twitter.util.Try +import javax.inject.Inject +import javax.inject.Singleton + +object UserAuthorEngagementRealTimeAggregateFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class UserAuthorEngagementRealTimeAggregateFeatureHydrator @Inject() ( + override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, + @Named(UserAuthorEngagementCache) override val client: ReadCache[(Long, Long), DataRecord], + @Named(RTAManhattanStore) mhClient: Option[ + ReadableStore[thrift.AggregationKey, DataRecord] + ], + override val statsReceiver: StatsReceiver) + extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[(Long, Long)] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserAuthorEngagementRealTimeAggregate") + + override val outputFeature: DataRecordInAFeature[TweetCandidate] = + UserAuthorEngagementRealTimeAggregateFeature + + override val aggregateGroups: Seq[AggregateGroup] = Seq( + userAuthorEngagementRealTimeAggregatesProd, + userAuthorShareEngagementsRealTimeAggregates + ) + + override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( + userAuthorEngagementRealTimeAggregatesProd -> "user-author.timelines.user_author_engagement_real_time_aggregates.", + userAuthorShareEngagementsRealTimeAggregates -> "user-author.timelines.user_author_share_engagements_real_time_aggregates." + ) + + def serializeKey(key: (Long, Long)): String = { + keyTransformD2(userIdFeature, authorIdFeature)(key) + } + + override def keysFromQueryAndCandidates( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[Option[(Long, Long)]] = { + val userId = query.getRequiredUserId + candidates.map { candidate => + CandidatesUtil + .getOriginalAuthorId(candidate.features) + .map((userId, _)) + } + } + + def convert(key: (Long, Long)): thrift.AggregationKey = { + val ak = keyTransformD2AggregationKey(userIdFeature, authorIdFeature)((key._1, key._2)) + thrift.AggregationKey( + ak.discreteFeaturesById, + ak.textFeaturesById + ) + } + + def fetchAndConstructDataRecordFromMh( + possiblyKeys: Seq[Option[(Long, Long)]] + ): Future[Seq[Try[DataRecord]]] = { + Future + .collect { + possiblyKeys.flatten + .map { + convert(_) + } + .grouped(64).map { keyGroup => + val results = mhClient.get.multiGet(keyGroup.toSet) + Future.collect(keyGroup.flatMap(results.get)).map { drSeq => + drSeq.map { drOpt => + if (drOpt.isEmpty) statsReceiver.scope("mhUserAuthorRTA").counter("empty").incr() + else statsReceiver.scope("mhUserAuthorRTA").counter("non_empty").incr() + Try(drOpt.map(postTransformer).getOrElse(EmptyDataRecord)) + } + } + }.toSeq + }.map(_.flatten) + } + + def getFetchFunc( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Future[Seq[Try[DataRecord]]] = { + val fetchFromMhOnly = query.params.getBoolean(EnableUserAuthorRTAMhOnlyParam) + val fetchFromMhAsFallBack = query.params.getBoolean(EnableUserAuthorRTAMhFallbackParam) + val possiblyKeys = keysFromQueryAndCandidates(query, candidates) + val stats = statsReceiver.scope("user_author_real_time_rta") + if (fetchFromMhOnly) { + fetchAndConstructDataRecordFromMh(possiblyKeys) + } else if (fetchFromMhAsFallBack) { + fetchAndConstructDataRecordsWithFallback( + possiblyKeys, + stats, + fetchAndConstructDataRecords, + fetchAndConstructDataRecordFromMh, + ) + } else { + fetchAndConstructDataRecords(possiblyKeys) + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + getFetchFunc(query, candidates).map { dataRecords => + val featureMaps = dataRecords.map { dataRecord => + FeatureMapBuilder().add(outputFeature, dataRecord).build() + } + featureMaps + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala new file mode 100644 index 000000000..a4c6b488d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala @@ -0,0 +1,159 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates + +import com.google.inject.name.Named +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.userIdFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableUserRTAMhFallbackParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnableUserRTAMhOnlyParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanStore +import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserEngagementCache +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.feature_hydrator.query.real_time_aggregates.BaseRealTimeAggregateQueryFeatureHydrator +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ReadCache +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKey +import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ +import com.twitter.util.Future +import com.twitter.util.Try +import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} +import javax.inject.Inject +import javax.inject.Singleton + +object UserEngagementRealTimeAggregateFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class UserEngagementRealTimeAggregatesFeatureHydrator @Inject() ( + @Named(UserEngagementCache) override val client: ReadCache[Long, DataRecord], + @Named(RTAManhattanStore) mhClient: Option[ReadableStore[thrift.AggregationKey, DataRecord]], + override val statsReceiver: StatsReceiver) + extends BaseRealTimeAggregateQueryFeatureHydrator[Long] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserEngagementRealTimeAggregates") + + override val outputFeature: DataRecordInAFeature[PipelineQuery] = + UserEngagementRealTimeAggregateFeature + + val aggregateGroups: Seq[AggregateGroup] = Seq( + userEngagementRealTimeAggregatesProd, + userShareEngagementsRealTimeAggregates, + userBCEDwellEngagementsRealTimeAggregates, + userEngagement48HourRealTimeAggregatesProd, + userNegativeEngagementAuthorUserState72HourRealTimeAggregates, + userNegativeEngagementAuthorUserStateRealTimeAggregates, + userProfileEngagementRealTimeAggregates, + ) + + override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( + userShareEngagementsRealTimeAggregates -> "user.timelines.user_share_engagements_real_time_aggregates.", + userBCEDwellEngagementsRealTimeAggregates -> "user.timelines.user_bce_dwell_engagements_real_time_aggregates.", + userEngagement48HourRealTimeAggregatesProd -> "user.timelines.user_engagement_48_hour_real_time_aggregates.", + userNegativeEngagementAuthorUserState72HourRealTimeAggregates -> "user.timelines.user_negative_engagement_author_user_state_72_hour_real_time_aggregates.", + userProfileEngagementRealTimeAggregates -> "user.timelines.user_profile_engagement_real_time_aggregates." + ) + + override def keysFromQueryAndCandidates(query: PipelineQuery): Option[Long] = { + Some(query.getRequiredUserId) + } + + val EmptyDataRecord = new DataRecord + + override def fetchAndConstructDataRecords( + possiblyKeys: Seq[Option[Long]] + ): Future[Seq[Try[DataRecord]]] = { + val keys = possiblyKeys.flatten.map { k => + val ak = AggregationKey(Map(userIdFeature -> k), Map.empty) + thrift.AggregationKey( + ak.discreteFeaturesById, + ak.textFeaturesById + ) + } + Future + .collect { + keys.map { key => + statsReceiver.scope("mhUserRTA").counter("mh_called").incr() + val result = mhClient.get.get(key) + result.map { drOpt => + if (drOpt.isEmpty) statsReceiver.scope("mhUserRTA").counter("empty").incr() + else statsReceiver.scope("mhUserRTA").counter("non_empty").incr() + Try(drOpt.map(postTransformer).getOrElse(EmptyDataRecord)) + } + } + } + } + + def convert(key: Long): thrift.AggregationKey = { + val ak = AggregationKey(Map(userIdFeature -> key), Map.empty) + thrift.AggregationKey( + ak.discreteFeaturesById, + ak.textFeaturesById + ) + } + + def fetchAndConstructDataRecordFromMh( + possiblyKeys: Seq[Option[Long]] + ): Future[Seq[Try[DataRecord]]] = { + Future + .collect { + possiblyKeys.flatten + .map { + convert(_) + } + .grouped(64).map { keyGroup => + val results = mhClient.get.multiGet(keyGroup.toSet) + Future.collect(keyGroup.flatMap(results.get)).map { drSeq => + drSeq.map { drOpt => + if (drOpt.isEmpty) statsReceiver.scope("mhUserRTA").counter("empty").incr() + else statsReceiver.scope("mhUserRTA").counter("non_empty").incr() + Try(drOpt.map(postTransformer).getOrElse(EmptyDataRecord)) + } + } + }.toSeq + }.map(_.flatten) + } + + def getFetchFunc( + query: PipelineQuery + ): Future[Seq[Try[DataRecord]]] = { + val fetchFromMhOnly = query.params.getBoolean(EnableUserRTAMhOnlyParam) + val fetchFromMhAsFallBack = query.params.getBoolean(EnableUserRTAMhFallbackParam) + val possiblyKeys = Seq(keysFromQueryAndCandidates(query)) + val stats = statsReceiver.scope("user_author_real_time_rta") + if (fetchFromMhOnly) { + fetchAndConstructDataRecordFromMh(possiblyKeys) + } else if (fetchFromMhAsFallBack) { + fetchAndConstructDataRecordsWithFallback( + possiblyKeys, + stats, + fetchAndConstructDataRecords, + fetchAndConstructDataRecordFromMh, + ) + } else { + fetchAndConstructDataRecords(possiblyKeys) + } + } + + override def hydrate( + query: PipelineQuery + ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { + getFetchFunc(query).map { dataRecords => + val featureMaps = dataRecords.map { dataRecord => + FeatureMapBuilder().add(outputFeature, dataRecord).build() + } + featureMaps.head + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserTweetTvVideoRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserTweetTvVideoRealTimeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..4e0305313 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserTweetTvVideoRealTimeAggregateFeatureHydrator.scala @@ -0,0 +1,96 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TvVideoByUserTweetCache +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.real_time_aggregates.BaseRealTimeAggregateBulkCandidateFeatureHydrator +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ReadCache +import com.twitter.stitch.Stitch +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup +import com.twitter.timelines.data_processing.ml_util.aggregation_framework.metrics.AggregateFeature +import com.twitter.timelines.prediction.common.aggregates.real_time.tv.TvOnlineAggregationFeaturesOnlyConfig +import com.twitter.util.Return +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +object DummyFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = { + new DataRecord() + } +} + +// The feature value is keyed by the RTA aggregation group prefix, then the AggregateFeature +// for ease of access. +object UserTweetTvVideoRealtimeAggregatesFeatures + extends Feature[TweetCandidate, Option[Map[String, Map[AggregateFeature[_], Double]]]] + +@Singleton +class UserTweetTvVideoRealTimeAggregateFeatureHydrator @Inject() ( + @Named(TvVideoByUserTweetCache) override val client: ReadCache[(Long, Long), DataRecord], + override val statsReceiver: StatsReceiver) + extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, Long), TweetCandidate] + with Conditionally[ScoredTweetsQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserTweetTvVideoRealTimeAggregate") + + override val outputFeature: DataRecordInAFeature[TweetCandidate] = DummyFeature + + override def onlyIf( + query: ScoredTweetsQuery + ): Boolean = query.videoType.contains(hmt.VideoType.LongForm) + + override def features: Set[Feature[_, _]] = Set( + UserTweetTvVideoRealtimeAggregatesFeatures + ) + + override val aggregateGroups: Seq[AggregateGroup] = Seq( + TvOnlineAggregationFeaturesOnlyConfig.userTweetViewRealTimeAggregates, + TvOnlineAggregationFeaturesOnlyConfig.userTweetPlaybackRealTimeAggregates, + TvOnlineAggregationFeaturesOnlyConfig.userTweetImpressionRealTimeAggregates + ) + + override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map.empty + + override def keysFromQueryAndCandidates( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[Option[(Long, Long)]] = { + candidates.map(candidate => + Some((query.getRequiredUserId, CandidatesUtil.getOriginalTweetId(candidate)))) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + fetchValues(keysFromQueryAndCandidates(query, candidates)).map { featureToAggValueMaps => + featureToAggValueMaps.map { featureToAggValueMap => + val value = featureToAggValueMap match { + case Return(r) => Some(r) + case _ => None + } + FeatureMapBuilder() + .add(UserTweetTvVideoRealtimeAggregatesFeatures, value) + .build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/BUILD.bazel new file mode 100644 index 000000000..604529661 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/BUILD.bazel @@ -0,0 +1,32 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/com/twitter/storehaus:core", + "configapi/configapi-decider", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "media-understanding/embeddings/src/main/thrift/com/twitter/media-understanding/embeddings:thrift-scala", + "servo/repo/src/main/scala", + "src/java/com/twitter/ml/api:api-base", + "src/scala/com/twitter/timelines/prediction/common/adapters:base", + "strato/config/columns/content_understanding:content_understanding-strato-client", + "strato/config/columns/tweetypie/managed:managed-strato-client", + "strato/config/columns/user-history-transformer/unhydrated-user-history:unhydrated-user-history-strato-client", + "strato/src/main/scala/com/twitter/strato/client", + "user_history_transformer/common/src/main/scala/com/twitter/user_history_transformer/util", + "user_history_transformer/thrift/src/main/thrift/com/twitter/user_history_transformer:user_history_transformer-scala", + ], + exports = [ + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/real_time_aggregates", + "timelines/data_processing/ml_util/aggregation_framework/real_time_aggregates", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/BaseUserHistoryEventsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/BaseUserHistoryEventsQueryFeatureHydrator.scala new file mode 100644 index 000000000..5a73e4732 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/BaseUserHistoryEventsQueryFeatureHydrator.scala @@ -0,0 +1,57 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.user_history + +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.BaseUserHistoryEventsAdapter +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.user_history_transformer.thriftscala.UnhydratedUserHistory +import com.twitter.user_history_transformer.thriftscala.UserHistory +import com.twitter.user_history_transformer.util.UserHistoryCompressionUtils +import scala.collection.JavaConverters._ + +object UserHistoryEventsFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +trait BaseUserHistoryEventsQueryFeatureHydrator extends QueryFeatureHydrator[PipelineQuery] { + private val DefaultDataRecord = new DataRecord() + + override def features: Set[Feature[_, _]] = Set(UserHistoryEventsFeature) + + def historyFetcher: Fetcher[Long, Unit, UnhydratedUserHistory] + def filterEvents(query: PipelineQuery, historyEvents: Seq[UserHistory]): Stitch[Seq[UserHistory]] + def adapter: BaseUserHistoryEventsAdapter + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + historyFetcher.fetch(query.getRequiredUserId).flatMap { result => + result.v match { + case Some(unhydratedUserHistory: UnhydratedUserHistory) => + val expandedValue: UnhydratedUserHistory = + UserHistoryCompressionUtils.expand(value = unhydratedUserHistory) + filterEvents(query, expandedValue.events) + .map { filteredEvents: Seq[UserHistory] => + val record = + if (filteredEvents.nonEmpty) + adapter + .adaptToDataRecords(filteredEvents) + .asScala + .headOption + .getOrElse(DefaultDataRecord) + else + DefaultDataRecord + FeatureMap(UserHistoryEventsFeature, record) + } + case None => + Stitch.value(FeatureMap(UserHistoryEventsFeature, DefaultDataRecord)) + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/ScoredTweetsUserHistoryEventsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/ScoredTweetsUserHistoryEventsQueryFeatureHydrator.scala new file mode 100644 index 000000000..f29d74e19 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/ScoredTweetsUserHistoryEventsQueryFeatureHydrator.scala @@ -0,0 +1,39 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.user_history + +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.BaseUserHistoryEventsAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.UserHistoryEventsAdapter +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.UserHistoryEventsLengthParam +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.user_history_transformer.unhydrated_user_history.UnhydratedUserHistoryMHProdClientColumn +import com.twitter.user_history_transformer.thriftscala.SourceType +import com.twitter.user_history_transformer.thriftscala.UnhydratedUserHistory +import com.twitter.user_history_transformer.thriftscala.UserHistory +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class ScoredTweetsUserHistoryEventsQueryFeatureHydrator @Inject() ( + unhydratedUserHistoryMHProdClientColumn: UnhydratedUserHistoryMHProdClientColumn) + extends BaseUserHistoryEventsQueryFeatureHydrator { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "ScoredTweetsUserHistoryEvents") + + override val historyFetcher: Fetcher[Long, Unit, UnhydratedUserHistory] = + unhydratedUserHistoryMHProdClientColumn.fetcher + + private val allowedSourceTypes: Set[SourceType] = Set(SourceType.Home, SourceType.Uua) + override val adapter: BaseUserHistoryEventsAdapter = UserHistoryEventsAdapter + override def filterEvents( + query: PipelineQuery, + historyEvents: Seq[UserHistory] + ): Stitch[Seq[UserHistory]] = { + val filteredEvents = historyEvents.filter { event => + allowedSourceTypes.contains(event.metadata.flatMap(_.source).getOrElse(SourceType.Uua)) + } + Stitch.value(filteredEvents.takeRight(query.params(UserHistoryEventsLengthParam))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/ScoredVideoTweetsUserHistoryEventsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/ScoredVideoTweetsUserHistoryEventsQueryFeatureHydrator.scala new file mode 100644 index 000000000..8a4166bf6 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/ScoredVideoTweetsUserHistoryEventsQueryFeatureHydrator.scala @@ -0,0 +1,339 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator.user_history + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.BaseUserHistoryEventsAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.VideoUserHistoryEventsAdapter +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClipClusterIdInMemCache +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClusterId95Store +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.UserHistoryEventsLengthParam +import com.twitter.mediaservices.commons.thriftscala.MediaCategory +import com.twitter.mediaservices.commons.tweetmedia.thriftscala.MediaInfo +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.columns.content_understanding.thriftscala.EntityWithMetadata +import com.twitter.strato.columns.content_understanding.thriftscala.TagWithMetadata +import com.twitter.strato.generated.client.content_understanding.DeserializedVideoAnnotationClientColumn +import com.twitter.strato.generated.client.tweetypie.managed.ImmersiveExploreClientEventsJobTesOnTweetClientColumn +import com.twitter.strato.generated.client.user_history_transformer.unhydrated_user_history.UnhydratedUserHistoryMHProdClientColumn +import com.twitter.unified_user_actions.thriftscala.ActionType +import com.twitter.user_history_transformer.thriftscala.SourceType +import com.twitter.user_history_transformer.thriftscala.UnhydratedUserHistory +import com.twitter.user_history_transformer.thriftscala.UserHistory +import com.twitter.user_history_transformer.thriftscala.UserHistoryMetadata +import javax.inject.Inject +import javax.inject.Singleton +import com.twitter.tweetypie.{thriftscala => tp} +import com.twitter.strato.catalog.Fetch +import javax.inject.Named +import com.twitter.storehaus.ReadableStore + +@Singleton +case class ScoredVideoTweetsUserHistoryEventsQueryFeatureHydrator @Inject() ( + immersiveExploreClientEventsJobTesOnTweetClientColumn: ImmersiveExploreClientEventsJobTesOnTweetClientColumn, + unhydratedUserHistoryMHProdClientColumn: UnhydratedUserHistoryMHProdClientColumn, + deserializedVideoAnnotationClientColumn: DeserializedVideoAnnotationClientColumn, + @Named(MediaClusterId95Store) clusterIdStore: ReadableStore[Long, Long], + @Named(MediaClipClusterIdInMemCache) mediaClipClusterIdInMemCache: InProcessCache[ + Long, + Option[Option[Long]] + ], + statsReceiver: StatsReceiver) + extends BaseUserHistoryEventsQueryFeatureHydrator + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "ScoredVideoTweetsUserHistoryEvents") + + override def onlyIf( + query: PipelineQuery + ): Boolean = + query.params(EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam) + + override val historyFetcher: Fetcher[Long, Unit, UnhydratedUserHistory] = + unhydratedUserHistoryMHProdClientColumn.fetcher + + val tesFetcher: Fetcher[Long, tp.GetTweetFieldsOptions, tp.GetTweetFieldsResult] = + immersiveExploreClientEventsJobTesOnTweetClientColumn.fetcher + + val TweetypieContentHydrationFields: Set[tp.TweetInclude] = Set( + tp.TweetInclude.TweetFieldId(tp.Tweet.CoreDataField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.IdField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.MediaField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.MediaKeysField.id) + ) + + private val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val immersiveClientActionTweetIdsCounter = + scopedStatsReceiver.counter("immersiveClientActionTweetIdsCounter") + private val validMediaInfoCounter = scopedStatsReceiver.counter("validMediaInfoCounter") + private val tesFetchCounter = statsReceiver.counter("tes_fetch_total") + private val tesFetchSuccessCounter = statsReceiver.counter("tes_fetch_success") + private val tesFetchFailureCounter = statsReceiver.counter("tes_fetch_failure") + private val combinedTweetIdsCounter = scopedStatsReceiver.counter("combinedTweetIdsCounter") + private val inValidMediaInfoCounter = scopedStatsReceiver.counter("inValidMediaInfoCounter") + private val inMemCacheHitCounter = scopedStatsReceiver.counter("cache/hit") + private val inMemCacheMissCounter = scopedStatsReceiver.counter("cache/miss") + + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") + private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") + + private val clusterIdFoundCounter = scopedStatsReceiver.counter("clusterIdFoundCounter") + private val clusterIdNotFoundCounter = scopedStatsReceiver.counter("clusterIdNotFoundCounter") + private val clusterIdFailureCounter = scopedStatsReceiver.counter("clusterIdFailureCountere") + + override val adapter: BaseUserHistoryEventsAdapter = VideoUserHistoryEventsAdapter + + override def filterEvents( + query: PipelineQuery, + historyEvents: Seq[UserHistory] + ): Stitch[Seq[UserHistory]] = { + + val immersiveActions = query match { + case query: PipelineQuery with ScoredTweetsQuery => + query.immersiveClientMetadata + .map(metadata => metadata.immersiveClientActions) + .getOrElse(Seq.empty) + .filter { action => action.tweetId.isDefined && action.watchTimeMs.isDefined } + case _ => + Seq.empty + } + + val immersiveActionMap: Map[Long, Int] = immersiveActions.map { action => + action.tweetId.get -> action.watchTimeMs.get + }.toMap + + val finalHistoryEvents = + historyEvents + .filter { event => + (event.actionType == ActionType.ClientTweetVideoWatchTime || + event.actionType == ActionType.ClientTweetVideoHeartbeat) && + !immersiveActionMap.contains(event.tweetId) + } + .takeRight(query.params(UserHistoryEventsLengthParam)) + + val combinedTweetIds = + (finalHistoryEvents.map(_.tweetId) ++ immersiveActions.map(_.tweetId.get)).distinct + combinedTweetIdsCounter.incr(combinedTweetIds.size) + + val fetchedData: Stitch[Map[Long, tp.GetTweetFieldsResult]] = + Stitch + .traverse(combinedTweetIds) { tweetId => + tesFetchCounter.incr() + tesFetcher + .fetch( + tweetId, + tp.GetTweetFieldsOptions(tweetIncludes = TweetypieContentHydrationFields) + ) + .map { + case Fetch.Result(Some(resolvedResult), _) => + tesFetchSuccessCounter.incr() + (tweetId, resolvedResult) + case _ => + tesFetchFailureCounter.incr() + ( + tweetId, + tp.GetTweetFieldsResult( + tweetId, + tp.TweetFieldsResultState.NotFound(tp.TweetFieldsResultNotFound()) + ) + ) + } + }.map(_.toMap) + + val grokMetadataFetched: Stitch[ + Map[Long, (Option[Seq[TagWithMetadata]], Option[Seq[EntityWithMetadata]])] + ] = + Stitch + .traverse(combinedTweetIds) { tweetId => + deserializedVideoAnnotationClientColumn.fetcher + .fetch(tweetId, Unit) + .map { + case Fetch.Result(response, _) => + if (response.nonEmpty) keyFoundCounter.incr() + else keyNotFoundCounter.incr() + ( + tweetId, + (response.flatMap(_.tags), response.flatMap(_.entities)) + ) + case _ => + keyFailureCounter.incr() + ( + tweetId, + (None: Option[List[TagWithMetadata]], None: Option[List[EntityWithMetadata]]) + ) + } + .handle { + case _ => + keyFailureCounter.incr() + ( + tweetId, + (None: Option[List[TagWithMetadata]], None: Option[List[EntityWithMetadata]]) + ) + } + }.map(_.toMap) + + val cluster95IdsFetched: Stitch[Map[Long, Long]] = + Stitch + .traverse(combinedTweetIds) { tweetId => + getFromCacheOrFetch(tweetId) + .map { + case Some(Some(clusterId)) => + clusterIdFoundCounter.incr() + (tweetId, clusterId) + case Some(None) | None => + clusterIdNotFoundCounter.incr() + (tweetId, -1L) + } + .handle { + case _ => + clusterIdFailureCounter.incr() + (tweetId, -1L) + } + }.map(_.toMap) + + for { + tweetResults <- fetchedData + grokMetadata <- grokMetadataFetched + clusterIds <- cluster95IdsFetched + } yield { + val tweetToMediaInfoMap = tweetResults.foldLeft( + Map.empty[Long, (Long, Option[MediaCategory], Option[Int], Option[Long], Long)] + ) { + case (mediaInfoMap, (tweetId, tweetResult)) => + tweetResult.tweetResult match { + case tp.TweetFieldsResultState.Found(found) => + val media = found.tweet.media.flatMap(_.headOption) + val authorId = found.tweet.coreData.map(_.userId) + val mediaId = media.map(_.mediaId) + val mediaCategory = media.flatMap(_.mediaKey).map(_.mediaCategory) + val videoDuration = media.flatMap(_.mediaInfo match { + case Some(MediaInfo.VideoInfo(videoInfo)) => Some(videoInfo.durationMillis) + case _ => None + }) + val clusterId = clusterIds.getOrElse(tweetId, -1L) + + mediaId match { + case Some(id) => + mediaInfoMap + (tweetId -> ( + ( + id, + mediaCategory, + videoDuration, + authorId, + clusterId + ) + )) + case None => mediaInfoMap + } + case _ => + mediaInfoMap + } + } + + val immersiveActionsUserHistory = immersiveActions.map { action => + val tweetId = action.tweetId.get + val (tagsOpt, entitiesOpt) = grokMetadata.getOrElse(tweetId, (None, None)) + val mediaInfo = tweetToMediaInfoMap.get(tweetId) + val finalTags = tagsOpt.getOrElse(List.empty[TagWithMetadata]) + val finalEntities = entitiesOpt.getOrElse(List.empty[EntityWithMetadata]) + val finalAuthorId: Option[Long] = mediaInfo.flatMap(_._4) + val finalClusterId: Long = mediaInfo.map(_._5).getOrElse(-1L) + + val (finalMediaId, finalMediaCategory, finalVideoDuration) = mediaInfo match { + case Some((mediaId, mediaCategory, videoDuration, _, _)) => + validMediaInfoCounter.incr() + (Some(mediaId), mediaCategory, videoDuration) + case None => + inValidMediaInfoCounter.incr() + (Some(-1L), Some(MediaCategory.TweetVideo), Some(-1)) + } + + UserHistory( + userId = query.getRequiredUserId, + tweetId = tweetId, + authorId = finalAuthorId, + actionType = ActionType.ClientTweetVideoWatchTime, + engagedTimestampMs = query.queryTime.inMilliseconds, + textTokens = None, + sourceTweetId = Some(tweetId), + metadata = Some( + UserHistoryMetadata( + source = Some(SourceType.ImmersiveVideo), + watchTime = Some(action.watchTimeMs.get.toDouble), + mediaId = finalMediaId, + mediaCategory = finalMediaCategory, + videoDuration = finalVideoDuration, + tags = Some(finalTags), + entities = Some(finalEntities), + clusterId = Some(finalClusterId) + ) + ) + ) + } + + immersiveClientActionTweetIdsCounter.incr(immersiveActionsUserHistory.size) + + (finalHistoryEvents ++ immersiveActionsUserHistory).map { event => + val tweetId = event.tweetId + val (tagsOpt, entitiesOpt) = grokMetadata.getOrElse(tweetId, (None, None)) + val mediaInfo = tweetToMediaInfoMap.get(tweetId) + val finalTags = tagsOpt.getOrElse(List.empty[TagWithMetadata]) + val finalEntities = entitiesOpt.getOrElse(List.empty[EntityWithMetadata]) + val finalAuthorId: Option[Long] = mediaInfo.flatMap(_._4) + val finalClusterId: Long = mediaInfo.map(_._5).getOrElse(-1L) + + val (finalMediaId, finalMediaCategory, finalVideoDuration) = mediaInfo match { + case Some((mediaId, mediaCategory, videoDuration, _, _)) => + validMediaInfoCounter.incr() + (Some(mediaId), mediaCategory, videoDuration) + case None => + inValidMediaInfoCounter.incr() + (Some(-1L), Some(MediaCategory.TweetVideo), Some(-1)) + } + + event.copy( + authorId = finalAuthorId.orElse(event.authorId), + metadata = event.metadata.map { metadata => + metadata.copy( + mediaId = finalMediaId, + mediaCategory = finalMediaCategory, + videoDuration = finalVideoDuration, + tags = Some(finalTags), + entities = Some(finalEntities), + clusterId = Some(finalClusterId) + ) + } + ) + } + } + } + + private def getFromCacheOrFetch(tweetId: Long): Stitch[Option[Option[Long]]] = { + mediaClipClusterIdInMemCache + .get(tweetId) + .map { cachedValue => + inMemCacheHitCounter.incr() + Stitch.value(cachedValue) + }.getOrElse { + inMemCacheMissCounter.incr() + Stitch + .callFuture(clusterIdStore.get(tweetId)) + .map { result => + val finalResult = Some(result) + mediaClipClusterIdInMemCache.set(tweetId, finalResult) + finalResult + } + .handle { + case _ => + None + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/AuthorDedupFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/AuthorDedupFilter.scala new file mode 100644 index 000000000..e0cee608b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/AuthorDedupFilter.scala @@ -0,0 +1,34 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Keep only 1 tweet per author + */ +object AuthorDedupFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("AuthorDedup") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val keptIds = candidates + .groupBy { candidate => + candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) + }.map { + case (_, candidates) => candidates.head.candidate.id + }.toSet + + val (kept, removed) = candidates.partition(c => keptIds.contains(c.candidate.id)) + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel index 59284224b..91c2423f6 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel @@ -5,21 +5,19 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "src/thrift/com/twitter/spam/rtf:safety-result-scala", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", + "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "stitch/stitch-socialgraph", - "stitch/stitch-tweetypie", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", + "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", "util/util-slf4j-api/src/main/scala", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ClipClusterDeduplicationFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ClipClusterDeduplicationFilter.scala new file mode 100644 index 000000000..1a30a63d5 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ClipClusterDeduplicationFilter.scala @@ -0,0 +1,59 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.ClipImageClusterIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaClusterIdsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +trait ClipClusterDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate] { + + val clusterIdFeature: Feature[TweetCandidate, Map[Long, Long]] + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (removedCandidateIds, _) = candidates.foldLeft((Set.empty[Long], Set.empty[Long])) { + case ((removedIds, seenClusterIds), candidate) => + val clusterIds = candidate.features + .getOrElse(clusterIdFeature, Map.empty[Long, Long]) + .values + .toSet + + if (clusterIds.size == 1) { + val clusterId = clusterIds.head + if (seenClusterIds.contains(clusterId)) { + (removedIds + candidate.candidate.id, seenClusterIds) + } else { + (removedIds, seenClusterIds + clusterId) + } + } else { + (removedIds, seenClusterIds) + } + } + + val (removed, kept) = candidates + .map(_.candidate) + .partition(c => removedCandidateIds.contains(c.id)) + + Stitch.value(FilterResult(kept = kept, removed = removed)) + } +} + +object ClipImageClusterDeduplicationFilter extends ClipClusterDeduplicationFilter { + override val identifier: FilterIdentifier = FilterIdentifier("ClipImageClusterDeduplication") + override val clusterIdFeature: Feature[TweetCandidate, Map[Long, Long]] = + ClipImageClusterIdsFeature +} + +object ClipVideoClusterDeduplicationFilter extends ClipClusterDeduplicationFilter { + override val identifier: FilterIdentifier = FilterIdentifier("ClipVideoClusterDeduplication") + override val clusterIdFeature: Feature[TweetCandidate, Map[Long, Long]] = + TweetMediaClusterIdsFeature +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ClusterBasedDedupFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ClusterBasedDedupFilter.scala new file mode 100644 index 000000000..4c1b8f07e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ClusterBasedDedupFilter.scala @@ -0,0 +1,138 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.UserHistoryEventsFeatures +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.VideoUserHistoryEventsFeatures +import com.twitter.home_mixer.functional_component.feature_hydrator.user_history.UserHistoryEventsFeature +import com.twitter.home_mixer.model.HomeFeatures.DedupClusterId88Feature +import com.twitter.home_mixer.model.HomeFeatures.DedupClusterIdFeature +import com.twitter.home_mixer.param.HomeGlobalParams.DedupHistoricalEventsTimeWindowParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnableClusterBased88DedupFilter +import com.twitter.home_mixer.param.HomeGlobalParams.EnableClusterBasedDedupFilter +import com.twitter.home_mixer.param.HomeGlobalParams.EnableNoClusterFilter +import scala.collection.convert.ImplicitConversions._ +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +object ClusterBasedDedupFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("ClusterBasedDedup") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = { + query.params(EnableClusterBasedDedupFilter) + } + + private def getViewedClusterIds(query: PipelineQuery): Set[Long] = { + val currentTimeMs = System.currentTimeMillis() + val dedupHistoricalEventsTimeWindow = query.params(DedupHistoricalEventsTimeWindowParam) + + val dataRecord = query.features.get + .getOrElse(UserHistoryEventsFeature, new DataRecord()) + + val clusterIds = dataRecord.getTensors match { + case tensors if tensors != null => + val clusterIdsTensor = + tensors.get(VideoUserHistoryEventsFeatures.VideoCluster95IdsFeature.getFeatureId) + val actionTimestampTensor = + tensors.get(UserHistoryEventsFeatures.ActionTimestampsFeature.getFeatureId) + + if (clusterIdsTensor != null && clusterIdsTensor.isSet() && + actionTimestampTensor != null && actionTimestampTensor.isSet()) { + val actionTimestampsBuffer = actionTimestampTensor.getInt64Tensor.longs + .map(_.longValue()) + val clusterIdsBuffer = clusterIdsTensor.getInt64Tensor.longs.map(_.longValue()) + + clusterIdsBuffer + .zip(actionTimestampsBuffer) + .collect { + case (clusterId, timestamp) + if timestamp > 0 && (currentTimeMs - timestamp) <= dedupHistoricalEventsTimeWindow => + clusterId + } + .toSet + } else { + Set.empty[Long] + } + case _ => + Set.empty[Long] + } + clusterIds + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val historicalClusters = getViewedClusterIds(query) // Still based on 0.95 threshold + val enable88Dedup = query.params(EnableClusterBased88DedupFilter) + val enableNoClusterFilter = query.params(EnableNoClusterFilter) + + val (kept, removed) = candidates.foldLeft( + ( + Seq[CandidateWithFeatures[TweetCandidate]](), // kept accumulator + Seq[CandidateWithFeatures[TweetCandidate]](), // removed accumulator + Set[Long](), // seen cluster IDs for 0.95 threshold + Set[Long]() // seen cluster IDs for 0.88 threshold + ) + ) { + case ((keptAcc, removedAcc, seenCluster95Ids, seenCluster88Ids), candidate) => + val clusterId95 = candidate.features + .getOrElse(DedupClusterIdFeature, None) // 0.95 threshold + val clusterId88 = candidate.features + .getOrElse(DedupClusterId88Feature, None) // 0.88 threshold + + (clusterId95, clusterId88) match { + case (Some(c95), _) if historicalClusters.contains(c95) => + // Historical match at 0.95 threshold, remove it + (keptAcc, removedAcc :+ candidate, seenCluster95Ids, seenCluster88Ids) + case (Some(c95), _) if seenCluster95Ids.contains(c95) => + // Duplicate at 0.95 threshold (non-historical), remove it + (keptAcc, removedAcc :+ candidate, seenCluster95Ids, seenCluster88Ids) + case (Some(c95), Some(c88)) if enable88Dedup && seenCluster88Ids.contains(c88) => + // Unique at 0.95 but duplicate at 0.88 threshold, remove it + (keptAcc, removedAcc :+ candidate, seenCluster95Ids + c95, seenCluster88Ids) + case (Some(c95), Some(c88)) => + // First occurrence at both levels, keep it + val updatedSeen88Ids = if (enable88Dedup) seenCluster88Ids + c88 else seenCluster88Ids + (keptAcc :+ candidate, removedAcc, seenCluster95Ids + c95, updatedSeen88Ids) + case (Some(c95), None) => + // Only 0.95 cluster ID, keep it if not seen + (keptAcc :+ candidate, removedAcc, seenCluster95Ids + c95, seenCluster88Ids) + case (None, Some(c88)) if enable88Dedup && seenCluster88Ids.contains(c88) => + // Only 0.88 cluster ID, remove if duplicate + (keptAcc, removedAcc :+ candidate, seenCluster95Ids, seenCluster88Ids) + case (None, Some(c88)) => + // Only 0.88 cluster ID, keep if not seen + val updatedSeen88Ids = if (enable88Dedup) seenCluster88Ids + c88 else seenCluster88Ids + (keptAcc :+ candidate, removedAcc, seenCluster95Ids, updatedSeen88Ids) + case (None, None) => + // No cluster IDs, keep or remove based on enableNoClusterFilter + if (enableNoClusterFilter) { + (keptAcc, removedAcc :+ candidate, seenCluster95Ids, seenCluster88Ids) // Remove it + } else { + (keptAcc :+ candidate, removedAcc, seenCluster95Ids, seenCluster88Ids) // Keep it + } + } + } match { + case (keptCandidates, removedCandidates, _, _) => + (keptCandidates, removedCandidates) + } + + Stitch.value( + FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ConsistentAspectRatioFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ConsistentAspectRatioFilter.scala new file mode 100644 index 000000000..b6d1a6eb3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ConsistentAspectRatioFilter.scala @@ -0,0 +1,57 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.VideoAspectRatioFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelines.configapi.Param + +/** + * Filter for making sure all video posts in a carousel has consistent aspect ratio + */ +case class ConsistentAspectRatioFilter( + allowVerticalVideosParam: Param[Boolean], + allowHorizontalVideosParam: Param[Boolean]) + extends Filter[PipelineQuery, TweetCandidate] { + + /** @see [[FilterIdentifier]] */ + override val identifier: FilterIdentifier = FilterIdentifier("ConsistentAspectRatio") + + /** + * Filter the list of candidates + * + * @return a FilterResult including both the list of kept candidate and the list of removed candidates + */ + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (hasAspectRatio, doesNotHaveAspectRatio) = candidates + .partition(_.features.getOrElse(VideoAspectRatioFeature, None).nonEmpty) + if (hasAspectRatio.nonEmpty) { + val isHorizontal = + (query.params(allowVerticalVideosParam), query.params(allowHorizontalVideosParam)) match { + case (false, _) => false + case (_, false) => true + case _ => // If both video types are allowed, take the first video's aspect ratio + val aspectRatioFirstVideo = + hasAspectRatio.head.features.getOrElse(VideoAspectRatioFeature, None).get + aspectRatioFirstVideo > 1.0 + } + val (consistentAspectRatio, differentAspectRatio) = hasAspectRatio.partition { candidate => + candidate.features.getOrElse(VideoAspectRatioFeature, None).get > 1.0 == isHorizontal + } + Stitch.value( + FilterResult( + kept = consistentAspectRatio.map(_.candidate), + removed = ((differentAspectRatio ++ doesNotHaveAspectRatio)).map(_.candidate))) + } else { + Stitch.value( + FilterResult(kept = Seq.empty, removed = doesNotHaveAspectRatio.map(_.candidate))) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/CountryFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/CountryFilter.scala new file mode 100644 index 000000000..0f0f3a825 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/CountryFilter.scala @@ -0,0 +1,67 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.home_mixer.param.HomeGlobalParams.EnableCountryFilter + +/** + * Filters tweets based on matching country location between user query and tweet candidates + */ +object CountryFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("Country") + + /** + * Determines if the filter should be applied based on the EnableCountryFilter parameter + */ + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = { + query.params(EnableCountryFilter) + } + + /** + * Filters candidates based on exact country location matching + * @param query The pipeline query containing user location features + * @param candidates The sequence of tweet candidates with their features + * @return A Stitch containing FilterResult with kept and removed candidates + */ + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val userCountryOpt = query.features + .map(_.getOrElse(LocationFeature, None)) + .flatMap(_.flatMap(_.country)) + + val (kept, removed) = candidates.partition { candidate => + val postLocationOpt = candidate.features.getOrElse(LocationFeature, None).flatMap(_.country) + val keep = (postLocationOpt, userCountryOpt) match { + case (Some(postLocation), Some(userCountry)) => + postLocation == userCountry + case (Some(_), None) => + false + case (None, _) => + true + case _ => + true + } + keep + } + + Stitch.value( + FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + )) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/CurrentPinnedTweetFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/CurrentPinnedTweetFilter.scala new file mode 100644 index 000000000..e857cdf46 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/CurrentPinnedTweetFilter.scala @@ -0,0 +1,31 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.CurrentPinnedTweetFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Keep only the currently pinned tweet per author + */ +object CurrentPinnedTweetFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("CurrentPinnedTweet") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (kept, removed) = candidates.partition { candidate => + val currentPinnedTweet = candidate.features.getOrElse(CurrentPinnedTweetFeature, None) + currentPinnedTweet.contains(candidate.candidate.id) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala index 582583e7f..b298fe5d3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala @@ -1,11 +1,11 @@ package com.twitter.home_mixer.functional_component.filter -import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature +import com.twitter.home_mixer.param.HomeGlobalParams.FeedbackFatigueFilteringDurationParam import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -32,8 +32,6 @@ object FeedbackFatigueFilter ): Boolean = query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) - private val DurationForFiltering = 14.days - override def apply( query: pipeline.PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] @@ -43,7 +41,7 @@ object FeedbackFatigueFilter .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) .filter { entry => val timeSinceFeedback = query.queryTime.minus(entry.timestamp) - timeSinceFeedback < DurationForFiltering && + timeSinceFeedback < query.params(FeedbackFatigueFilteringDurationParam) && entry.feedbackType == tls.FeedbackType.SeeFewer }.groupBy(_.engagementType) @@ -84,6 +82,6 @@ object FeedbackFatigueFilter feedbackEntries: Seq[FeedbackEntry], ): Set[Long] = feedbackEntries.collect { - case FeedbackEntry(_, _, FeedbackEntity.UserId(userId), _, _) => userId + case FeedbackEntry(_, _, FeedbackEntity.UserId(userId), _, _, _) => userId }.toSet } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokGoreFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokGoreFilter.scala new file mode 100644 index 000000000..6aaff007c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokGoreFilter.scala @@ -0,0 +1,39 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.param.HomeGlobalParams.EnableGrokGoreFilter +import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Filter out gore tweets based on grok annotations + */ +object GrokGoreFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("GrokGore") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableGrokGoreFilter) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (removed, kept) = candidates.partition { candidate => + val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) + annotations.flatMap(_.metadata.map(_.isGore)).getOrElse(false) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokNsfwFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokNsfwFilter.scala new file mode 100644 index 000000000..a3b6cf8d7 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokNsfwFilter.scala @@ -0,0 +1,47 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.param.HomeGlobalParams.EnableNsfwFilter +import com.twitter.home_mixer.param.HomeGlobalParams.EnableSoftNsfwFilter +import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Filter out NSFW tweets based on grok annotations + */ +object GrokNsfwFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("GrokNsfw") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableNsfwFilter) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (removed, kept) = candidates.partition { candidate => + val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) + val isNsfw = annotations.flatMap(_.metadata.map(_.isNsfw)).getOrElse(false) + val isSoftNsfw = annotations.flatMap(_.metadata.map(_.isSoftNsfw)).getOrElse(false) + + if (query.params(EnableSoftNsfwFilter)) { + isNsfw || isSoftNsfw + } else { + isNsfw + } + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokSpamFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokSpamFilter.scala new file mode 100644 index 000000000..53a53fac6 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokSpamFilter.scala @@ -0,0 +1,39 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.param.HomeGlobalParams.EnableGrokSpamFilter +import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Filter out spam tweets based on grok annotations + */ +object GrokSpamFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("GrokSpam") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableGrokSpamFilter) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (removed, kept) = candidates.partition { candidate => + val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) + annotations.flatMap(_.metadata.map(_.isSpam)).getOrElse(false) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokViolentFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokViolentFilter.scala new file mode 100644 index 000000000..fc8d44d10 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokViolentFilter.scala @@ -0,0 +1,39 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.param.HomeGlobalParams.EnableGrokViolentFilter +import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Filter out violent tweets based on grok annotations + */ +object GrokViolentFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("GrokViolent") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableGrokViolentFilter) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (removed, kept) = candidates.partition { candidate => + val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) + annotations.flatMap(_.metadata.map(_.isViolent)).getOrElse(false) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/HasAuthorFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/HasAuthorFilter.scala new file mode 100644 index 000000000..28dc99f06 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/HasAuthorFilter.scala @@ -0,0 +1,25 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object HasAuthorFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("HasAuthor") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (kept, removed) = candidates.partition { candidate => + candidate.features.getOrElse(AuthorIdFeature, None).isDefined + } + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/HasMultipleMediaFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/HasMultipleMediaFilter.scala new file mode 100644 index 000000000..abfc8fdac --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/HasMultipleMediaFilter.scala @@ -0,0 +1,35 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.HasMultipleMedia +import com.twitter.home_mixer.param.HomeGlobalParams.EnableHasMultipleMediaFilter +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object HasMultipleMediaFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("HasMultipleMedia") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = { + query.params(EnableHasMultipleMediaFilter) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (removed, kept) = candidates.partition { candidate => + candidate.features.getOrElse(HasMultipleMedia, false) + } + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala index 285eec49f..94a2ca133 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala @@ -3,19 +3,23 @@ package com.twitter.home_mixer.functional_component.filter import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.tracing.Trace import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import com.twitter.util.logging.Logging - import javax.inject.Inject import javax.inject.Singleton +import scala.collection.immutable.ListSet /** * Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author @@ -33,9 +37,13 @@ case class InvalidSubscriptionTweetFilter @Inject() ( override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet") private val scopedStatsReceiver = statsReceiver.scope(identifier.toString) + private val servedTypeStatsReceiver = scopedStatsReceiver.scope("ServedType") private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet") private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet") + private val forYouScoredTweetsCandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouScoredTweets") + override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] @@ -52,7 +60,22 @@ case class InvalidSubscriptionTweetFilter @Inject() ( Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)), ) socialGraphClient.exists(request).map(_.exists).map { valid => - if (!valid) invalidCounter.incr() else validCounter.incr() + if (!valid) { + invalidCounter.incr() + val candidatePipelines = candidate.features + .getOrElse(CandidatePipelines, ListSet.empty[CandidatePipelineIdentifier]) + // Temporary debugging code + if (candidatePipelines.contains(forYouScoredTweetsCandidatePipelineIdentifier)) { + val servedType = + candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) + servedTypeStatsReceiver.counter(servedType.name).incr() + logger.info( + s"Removing subscription Tweet ${candidate.candidate.id} " + + s"for User ${query.getRequiredUserId} from Author $exclusiveAuthorId " + + s"with traceId: ${Trace.id.traceId.toLong}" + ) + } + } else validCounter.incr() valid } } else Stitch.value(true) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/LocationFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/LocationFilter.scala new file mode 100644 index 000000000..013f1dccc --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/LocationFilter.scala @@ -0,0 +1,34 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object LocationFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("Location") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val userLocationOpt = query.features.get.getOrElse(LocationFeature, None) + + val (kept, removed) = candidates.partition { candidate => + val postLocationOpt = candidate.features.getOrElse(LocationIdFeature, None) + (postLocationOpt, userLocationOpt) match { + case (Some(postLocation), Some(userLocation)) => userLocation.matches(postLocation) + case (Some(postLocation), None) => false + case _ => true + } + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MaxVideoDurationFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MaxVideoDurationFilter.scala new file mode 100644 index 000000000..77f650a68 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MaxVideoDurationFilter.scala @@ -0,0 +1,44 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableMaxVideoDurationFilter +import com.twitter.home_mixer.param.HomeGlobalParams.MaxVideoDurationThresholdParam +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Filter out tweets based on video duration + */ +object MaxVideoDurationFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("MaxVideoDuration") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableMaxVideoDurationFilter) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val maxVideoDurationThresh = query.params(MaxVideoDurationThresholdParam) + val (kept, removed) = candidates.partition { candidate => + val hasVideoFeature = candidate.features.getOrElse(HasVideoFeature, false) + val videoDuration = candidate.features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) + + hasVideoFeature && (videoDuration <= maxVideoDurationThresh) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MediaDeduplicationFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MediaDeduplicationFilter.scala new file mode 100644 index 000000000..c9c635b5a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MediaDeduplicationFilter.scala @@ -0,0 +1,42 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Remove duplicate media in the same payload + */ +object MediaIdDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("MediaDeduplication") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (removedCandidateIds, _) = candidates.foldLeft((Set.empty[Long], Set.empty[Long])) { + case ((removedIds, seenMediaIds), candidate) => + val mediaIds = candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty[Long]) + if (mediaIds.size == 1) { + val mediaId = mediaIds.head + if (seenMediaIds.contains(mediaId)) { + (removedIds + candidate.candidate.id, seenMediaIds) + } else { + (removedIds, seenMediaIds + mediaId) + } + } else { + (removedIds, seenMediaIds) + } + } + + val (removed, kept) = candidates.map(_.candidate) + .partition(c => removedCandidateIds.contains(c.id)) + Stitch.value(FilterResult(kept = kept, removed = removed)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MinVideoDurationFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MinVideoDurationFilter.scala new file mode 100644 index 000000000..d1b20619e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MinVideoDurationFilter.scala @@ -0,0 +1,61 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaCompletionRateFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableMinVideoDurationFilter +import com.twitter.home_mixer.param.HomeGlobalParams.MinVideoDurationThresholdParam +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Filter out tweets based on video duration + */ +object MinVideoDurationFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("MinVideoDuration") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableMinVideoDurationFilter) + + private val CompletionRateThreshold = 90 + private val FavThreshold = 30 + private val LongDuration = 5000 + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val minVideoDurationThresh = query.params(MinVideoDurationThresholdParam) + val (removed, kept) = candidates.partition { candidate => + val hasVideoFeature = candidate.features.getOrElse(HasVideoFeature, false) + val completionRate = + candidate.features.getOrElse(TweetMediaCompletionRateFeature, None).getOrElse(0.0) + val videoDuration = candidate.features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) + val ebFeatures = candidate.features.getOrElse(EarlybirdFeature, None) + val favCount = ebFeatures.flatMap(_.favCountV2) + + val sketchyCompletionRate = + favCount.forall(_ > FavThreshold) && + completionRate > CompletionRateThreshold && + videoDuration > LongDuration + + val lowDuration = videoDuration <= minVideoDurationThresh + + hasVideoFeature && (lowDuration || sketchyCompletionRate) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenMediaIdsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenMediaIdsFilter.scala new file mode 100644 index 000000000..6bfea4310 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenMediaIdsFilter.scala @@ -0,0 +1,38 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.ImpressedMediaIds +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Filter out users' previously seen mediaIds + */ +object PreviouslySeenMediaIdsFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("PreviouslySeenMediaIds") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val seenMediaIds = + query.features.map(_.getOrElse(ImpressedMediaIds, Seq.empty)).toSet.flatten + + val (removed, kept) = candidates.partition { candidate => + candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty).exists(seenMediaIds.contains) + } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.scala index 047233a41..f530b65e5 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.scala @@ -1,7 +1,9 @@ package com.twitter.home_mixer.functional_component.filter +import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature +import com.twitter.home_mixer.model.HomeFeatures.UserRecentEngagementTweetIdsFeature +import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.home_mixer.util.TweetImpressionsHelper import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult @@ -9,27 +11,35 @@ import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch +import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilterItem /** - * Filter out users' previously seen tweets from 2 sources: - * 1. Heron Topology Impression Store in Memcache; - * 2. Manhattan Impression Store; + * Filter out users' previously seen tweets from Impression Bloom Filter */ -object PreviouslySeenTweetsFilter extends Filter[PipelineQuery, TweetCandidate] { +object PreviouslySeenTweetsFilter + extends Filter[PipelineQuery with HasSeenTweetIds, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("PreviouslySeenTweets") override def apply( - query: PipelineQuery, + query: PipelineQuery with HasSeenTweetIds, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { + val bloomFilterSeq = query.features.map(_.get(ImpressionBloomFilterFeature)).get + val bloomFilters = + bloomFilterSeq.entries.map(ImpressionBloomFilterItem.fromThrift(_).bloomFilter) - val seenTweetIds = - query.features.map(TweetImpressionsHelper.tweetImpressions).getOrElse(Set.empty) + val seenTweetIds = query.seenTweetIds.getOrElse(Seq.empty).toSet + val engagedTweetIds = + query.features + .map(_.getOrElse(UserRecentEngagementTweetIdsFeature, Seq.empty).toSet) + .getOrElse(Set.empty) val (removed, kept) = candidates.partition { candidate => - val tweetIdAndSourceId = CandidatesUtil.getTweetIdAndSourceId(candidate) - tweetIdAndSourceId.exists(seenTweetIds.contains) + CandidatesUtil.getTweetIdAndSourceId(candidate).exists { tweetId => + seenTweetIds.contains(tweetId) || engagedTweetIds.contains(tweetId) || + bloomFilters.exists(filter => filter.mayContain(tweetId)) + } } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.scala index 56b6a1c0b..1bda17c8d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.scala @@ -2,6 +2,7 @@ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableServedFilterAllRequests import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter @@ -21,14 +22,14 @@ object PreviouslyServedTweetsFilter query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = { - query.features.exists(_.getOrElse(GetOlderFeature, false)) + query.features.exists(_.getOrElse(GetOlderFeature, false)) || + query.params(EnableServedFilterAllRequests) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { - val servedTweetIds = query.features.map(_.getOrElse(ServedTweetIdsFeature, Seq.empty)).toSeq.flatten.toSet diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/QuoteDeduplicationFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/QuoteDeduplicationFilter.scala new file mode 100644 index 000000000..18adf7a77 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/QuoteDeduplicationFilter.scala @@ -0,0 +1,30 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Remove base tweet if it is quoted in another candidate + */ +object QuoteDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("QuoteDeduplication") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val quotedTweets = candidates.flatMap(_.features.getOrElse(QuotedTweetIdFeature, None)).toSet + + val (removed, kept) = + candidates.map(_.candidate).partition(candidate => quotedTweets.contains(candidate.id)) + + Stitch.value(FilterResult(kept = kept, removed = removed)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RegionFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RegionFilter.scala new file mode 100644 index 000000000..e663e790d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RegionFilter.scala @@ -0,0 +1,57 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.home_mixer.param.HomeGlobalParams.EnableRegionFilter + +/** + * Filters tweets based on matching region location between user query and tweet candidates + */ +object RegionFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("Region") + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = { + query.params(EnableRegionFilter) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val userRegionOpt = query.features + .map(_.getOrElse(LocationFeature, None)) + .flatMap(_.flatMap(_.region)) + + val (kept, removed) = candidates.partition { candidate => + val postLocationOpt = candidate.features.getOrElse(LocationFeature, None).flatMap(_.region) + val keep = (postLocationOpt, userRegionOpt) match { + case (Some(postLocation), Some(userRegion)) => + postLocation == userRegion + case (Some(_), None) => + false + case (None, _) => + true + case _ => + true + } + keep + } + + Stitch.value( + FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + )) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala index a1b99df66..276f78dbb 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala @@ -16,17 +16,10 @@ object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] { query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { + val (kept, removed) = candidates.partition { candidate => + candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty + } - val (kept, removed) = candidates - .partition { candidate => - candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty - } - - val filterResult = FilterResult( - kept = kept.map(_.candidate), - removed = removed.map(_.candidate) - ) - - Stitch.value(filterResult) + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.scala index 1e1f7f03a..258786a74 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.scala @@ -38,8 +38,8 @@ object RetweetDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate] } val (kept, removed) = - candidates - .map(_.candidate).partition(candidate => dedupedTweetIdsSet.contains(candidate.id)) + candidates.map(_.candidate).partition(candidate => dedupedTweetIdsSet.contains(candidate.id)) + Stitch.value(FilterResult(kept = kept, removed = removed)) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/SlopFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/SlopFilter.scala new file mode 100644 index 000000000..a1d7911ad --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/SlopFilter.scala @@ -0,0 +1,68 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.core_workflows.user_model.{thriftscala => um} +import com.twitter.home_mixer.model.HomeFeatures.AuthorFollowersFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.SlopAuthorFeature +import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableSlopFilter +import com.twitter.home_mixer.param.HomeGlobalParams.EnableSlopFilterEligibleUserStateParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnableSlopFilterLowSignalUsers +import com.twitter.home_mixer.param.HomeGlobalParams.SlopMinFollowers +import com.twitter.home_mixer.util.SignalUtil +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object SlopFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("Slop") + + private val MinFollowingThreshold = 5 + private val EligibleUserStates: Set[um.UserState] = + Set(um.UserState.NearZero, um.UserState.New, um.UserState.VeryLight) + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = { + val numSlopAuthorsFollowed = candidates + .filter { candidate => + candidate.features.getOrElse(SlopAuthorFeature, false) && + candidate.features.getOrElse(InNetworkFeature, false) + }.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct.size + + val userState = query.features.flatMap(_.getOrElse(UserStateFeature, None)) + + val lowSignalUser = SignalUtil.isLowSignalUser(query) + + numSlopAuthorsFollowed < MinFollowingThreshold && + ( + (userState.forall(EligibleUserStates.contains) && + query.params(EnableSlopFilterEligibleUserStateParam)) || + (lowSignalUser && query.params(EnableSlopFilterLowSignalUsers)) || + query.params(EnableSlopFilter) + ) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val minFollowersThreshold = query.params(SlopMinFollowers) + + val (removed, kept) = candidates.partition { candidate => + candidate.features.getOrElse(SlopAuthorFeature, false) && + !candidate.features.getOrElse(InNetworkFeature, false) && + candidate.features.getOrElse(AuthorFollowersFeature, None).forall(_ > minFollowersThreshold) + } + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/TweetHydrationFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/TweetHydrationFilter.scala new file mode 100644 index 000000000..467cb074d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/TweetHydrationFilter.scala @@ -0,0 +1,29 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Filter out tweets that fail Tweetypie VF hydration + */ +object TweetHydrationFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("TweetHydration") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (kept, removed) = candidates.partition { candidate => + candidate.features.getOrElse(IsHydratedFeature, false) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/WeeklyBookmarkFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/WeeklyBookmarkFilter.scala new file mode 100644 index 000000000..a361db7db --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/WeeklyBookmarkFilter.scala @@ -0,0 +1,34 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.model.HomeFeatures.BookmarkedTweetTimestamp +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.util.Time + +object WeeklyBookmarkFilter extends Filter[PipelineQuery, TweetCandidate] { + override val identifier: FilterIdentifier = FilterIdentifier("WeeklyBookmark") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (keptCandidates, removedCandidates) = candidates.partition { filterCandidate => + filterCandidate.features + .get(BookmarkedTweetTimestamp).exists { timestamp => + val aWeekAgo = Time.now - 7.days + Time.fromMilliseconds(timestamp) >= aWeekAgo + } + } + + Stitch.value( + FilterResult( + kept = keptCandidates.map(_.candidate), + removed = removedCandidates.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/AllowForYouRecommendationsGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/AllowForYouRecommendationsGate.scala new file mode 100644 index 000000000..8042b2571 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/AllowForYouRecommendationsGate.scala @@ -0,0 +1,24 @@ +package com.twitter.home_mixer.functional_component.gate + +import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsForYouRecommendationsFeature +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * This gate disables out of network candidate pipelines when the AllowForYouRecommendations + * user preference is set to false. + * Defaults to true if the preference is not set. + */ +object AllowForYouRecommendationsGate extends Gate[PipelineQuery] { + override val identifier: GateIdentifier = GateIdentifier("AllowForYouRecommendations") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + val allowForYouRecommendations = query.features + .flatMap(_.getOrElse(ViewerAllowsForYouRecommendationsFeature, Some(true))).getOrElse( + true + ) + Stitch.value(allowForYouRecommendations) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel index 6be06dee9..b90dedf54 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel @@ -4,16 +4,12 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "stitch/stitch-socialgraph", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module:test-user-mapper", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BookmarksTimeGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BookmarksTimeGate.scala new file mode 100644 index 000000000..771b1b628 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BookmarksTimeGate.scala @@ -0,0 +1,38 @@ +package com.twitter.home_mixer.functional_component.gate + +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableBookmarksModuleWeekendGate +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import java.time.DayOfWeek +import java.time.ZonedDateTime +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class BookmarksTimeGate @Inject() (serviceIdentifier: ServiceIdentifier) + extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("BookmarksTime") + + // Serve on the weekends, with 48 hours injection time since persistence store is only 2 days. This way we have 2 + // chances to show the module both on Saturday and Sunday + private def isWeekend(zonedDateTime: ZonedDateTime) = { + zonedDateTime.getDayOfWeek match { + case DayOfWeek.SATURDAY => true + case DayOfWeek.SUNDAY => true + case _ => false + } + + } + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + val gateDisabled = !query.params(EnableBookmarksModuleWeekendGate) + val isDevel = serviceIdentifier.environment.toLowerCase != "prod" + val isDarkRequest = query.features.flatMap { _.get(HasDarkRequestFeature) }.getOrElse(false) + Stitch.value( + gateDisabled || isDarkRequest || isDevel || isWeekend(query.queryTime.toZonedDateTime)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSyntheticUserGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSyntheticUserGate.scala new file mode 100644 index 000000000..9dde1cdf9 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSyntheticUserGate.scala @@ -0,0 +1,22 @@ +package com.twitter.home_mixer.functional_component.gate + +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * A Synthetic User is a user who is created and managed on behalf of XCC's + * Loadtesting framework. This gate can be used to turn off certain functionality like ads for + * these users. + */ +object ExcludeSyntheticUserGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("ExcludeSyntheticUser") + private val STRESS_TEST_APP_ID: Long = 1L + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + val isSyntheticUser = query.clientContext.appId.contains(STRESS_TEST_APP_ID) + Stitch.value(!isSyntheticUser) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/PersistenceStoreDurationValidationGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/PersistenceStoreDurationValidationGate.scala new file mode 100644 index 000000000..d0fee7a92 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/PersistenceStoreDurationValidationGate.scala @@ -0,0 +1,53 @@ +package com.twitter.home_mixer.functional_component.gate + +import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature +import com.twitter.product_mixer.core.functional_component.configapi.StaticParam +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelinemixer.injection.store.persistence.TimelinePersistenceUtils +import com.twitter.timelines.configapi.Param +import com.twitter.timelines.util.client_info.ClientPlatform +import com.twitter.util.Duration +import com.twitter.util.Time + +/** + * For users who max out the persistence store, we may serve certain modules too frequently. + * Use this gate to prevent that. + * + * Gate stops the request if the time since the oldest entry < input duration + * (or if there aren't enough entries, which can also cause a small duration) + * + * @param minInjectionIntervalParam the desired minimum interval between injections + */ +case class PersistenceStoreDurationValidationGate( + minDuration: Param[Duration] = StaticParam(48.hours)) + extends Gate[PipelineQuery] + with TimelinePersistenceUtils { + + override val identifier: GateIdentifier = GateIdentifier("PersistenceStoreDurationValidation") + + private val MinEntries = 1500 + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + val continue = query.features.map { featureMap => + val timelineResponses = featureMap.getOrElse(PersistenceEntriesFeature, Seq.empty) + val clientPlatform = ClientPlatform.fromQueryOptions( + clientAppId = query.clientContext.appId, + userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString) + ) + val sortedResponses = responseByClient(clientPlatform, timelineResponses) + val entryCount = sortedResponses.flatMap(_.entries).size + + val oldestTime = sortedResponses.lastOption.map(_.servedTime).getOrElse(Time.Bottom) + + val duration = Time.now.since(oldestTime) + duration > query.params(minDuration) || entryCount < MinEntries + } + + Stitch.value(continue.getOrElse(true)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RateLimitGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RateLimitGate.scala new file mode 100644 index 000000000..b9af7815a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RateLimitGate.scala @@ -0,0 +1,19 @@ +package com.twitter.home_mixer.functional_component.gate + +import com.twitter.home_mixer.model.HomeFeatures.ViewerHasPremiumTier +import com.twitter.home_mixer.model.HomeFeatures.ViewerIsRateLimited +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object RateLimitGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("RateLimit") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + val isRateLimited = query.features.map(_.getOrElse(ViewerIsRateLimited, false)) + val hasPremiumTier = query.features.map(_.getOrElse(ViewerHasPremiumTier, false)) + Stitch.value(isRateLimited.contains(false) || hasPremiumTier.contains(true)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RateLimitNotGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RateLimitNotGate.scala new file mode 100644 index 000000000..d293274d2 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RateLimitNotGate.scala @@ -0,0 +1,19 @@ +package com.twitter.home_mixer.functional_component.gate + +import com.twitter.home_mixer.model.HomeFeatures.ViewerHasPremiumTier +import com.twitter.home_mixer.model.HomeFeatures.ViewerIsRateLimited +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object RateLimitNotGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("RateLimitNot") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + val isRateLimited = query.features.map(_.getOrElse(ViewerIsRateLimited, false)) + val hasPremiumTier = query.features.map(_.getOrElse(ViewerHasPremiumTier, false)) + Stitch.value(isRateLimited.contains(true) && hasPremiumTier.contains(false)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RecentlyServedByServedTypeGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RecentlyServedByServedTypeGate.scala new file mode 100644 index 000000000..ff38a3ec1 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RecentlyServedByServedTypeGate.scala @@ -0,0 +1,58 @@ +package com.twitter.home_mixer.functional_component.gate + +import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelinemixer.injection.store.persistence.TimelinePersistenceUtils +import com.twitter.timelines.configapi.Param +import com.twitter.timelineservice.model.rich.EntityIdType +import com.twitter.timelineservice.model.TweetScoreV1 +import com.twitter.util.Duration +import com.twitter.util.Time +import com.twitter.home_mixer.{thriftscala => hmt} + +/** + * Gate used to reduce the frequency of injections based on specific served types. + * This gate checks if any tweets in the persistence store have a served type equal to the specified targetServedType. + * Note that the actual interval between injections may be less than the specified minInjectionIntervalParam + * if data is unavailable or missing. For example, being deleted by the persistence store via a TTL or similar mechanism. + * + * @param minInjectionIntervalParam the desired minimum interval between injections + * @param targetServedType the served type to check for in persisted tweets + */ +case class RecentlyServedByServedTypeGate( + minInjectionIntervalParam: Param[Duration], + targetServedType: hmt.ServedType) + extends Gate[PipelineQuery] + with TimelinePersistenceUtils { + + override val identifier: GateIdentifier = GateIdentifier("RecentlyServedByServedType") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = + Stitch( + query.queryTime.since(getLastInjectionTime(query)) > query.params(minInjectionIntervalParam)) + + private def getLastInjectionTime(query: PipelineQuery) = query.features + .flatMap { featureMap => + val timelineResponses = featureMap.getOrElse(PersistenceEntriesFeature, Seq.empty) + val latestResponseWithTargetServedTypeEntry = + timelineResponses.find { response => + response.entries.exists { entry => + entry.entityIdType == EntityIdType.Tweet && + entry.itemIds.exists { itemIds => + itemIds.exists { itemId => + itemId.tweetScore.exists { + case tweetScore: TweetScoreV1 => + tweetScore.servedType.contains(targetServedType.originalName) + case _ => false + } + } + } + } + } + + latestResponseWithTargetServedTypeEntry.map(_.servedTime) + }.getOrElse(Time.Bottom) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TestUserProbabilisticGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TestUserProbabilisticGate.scala new file mode 100644 index 000000000..e41140be9 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TestUserProbabilisticGate.scala @@ -0,0 +1,33 @@ +package com.twitter.home_mixer.functional_component.gate + +import javax.inject.Inject +import javax.inject.Singleton + +import com.twitter.product_mixer.component_library.module.TestUserMapper +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Generic gate used to assign a probability that the associated Component / Pipeline operates + * on a Synthetic test user. + * + * @param testUserMapper the testUserMapper utility object that evaluates if this is a test user + */ +@Singleton +class TestUserProbabilisticGate @Inject() (testUserMapper: TestUserMapper) + extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("TestUserProbabilistic") + private val TEST_USERS_GATE_PROBABILITY = 0.05 + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + if (!testUserMapper.isTestUser( + query.clientContext) || math.random < TEST_USERS_GATE_PROBABILITY) { + Stitch.True + } else { + Stitch.False + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.scala index 31fff2306..860281b0a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.scala @@ -1,12 +1,11 @@ package com.twitter.home_mixer.functional_component.gate import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent -import com.twitter.product_mixer.core.feature.Feature +import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 import com.twitter.timelinemixer.injection.store.persistence.TimelinePersistenceUtils import com.twitter.timelines.configapi.Param import com.twitter.timelines.util.client_info.ClientPlatform @@ -24,7 +23,6 @@ import com.twitter.util.Time */ case class TimelinesPersistenceStoreLastInjectionGate( minInjectionIntervalParam: Param[Duration], - persistenceEntriesFeature: Feature[PipelineQuery, Seq[TimelineResponseV3]], entityIdType: EntityIdType.Value) extends Gate[PipelineQuery] with TimelinePersistenceUtils { @@ -37,7 +35,7 @@ case class TimelinesPersistenceStoreLastInjectionGate( private def getLastInjectionTime(query: PipelineQuery) = query.features .flatMap { featureMap => - val timelineResponses = featureMap.getOrElse(persistenceEntriesFeature, Seq.empty) + val timelineResponses = featureMap.getOrElse(PersistenceEntriesFeature, Seq.empty) val clientPlatform = ClientPlatform.fromQueryOptions( clientAppId = query.clientContext.appId, userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel index 99605204c..8f698dd6c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel @@ -3,14 +3,20 @@ scala_library( compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ + "3rdparty/jvm/io/grpc:grpc-protobuf", + "3rdparty/jvm/io/grpc:grpc-stub", + "finagle-internal/finagle-grpc/src/main/scala", + "finatra/inject/inject-utils/src/main/scala", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/thrift/com/twitter/timelines/common:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module:test-user-mapper", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "timelines/src/main/scala/com/twitter/timelines/clients/predictionservice", "timelineservice/common:model", + "user_history_transformer/service/src/main/java/com/x/user_action_sequence", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala index ceb71139e..075cfb014 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala @@ -35,7 +35,6 @@ object FeedbackFatigueScorer override def onlyIf(query: PipelineQuery): Boolean = query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) - val DurationForFiltering = 14.days val DurationForDiscounting = 140.days private val ScoreMultiplierLowerBound = 0.2 private val ScoreMultiplierUpperBound = 1.0 @@ -54,7 +53,7 @@ object FeedbackFatigueScorer .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) .filter { entry => val timeSinceFeedback = query.queryTime.minus(entry.timestamp) - timeSinceFeedback < DurationForFiltering + DurationForDiscounting && + timeSinceFeedback < DurationForDiscounting && entry.feedbackType == tls.FeedbackType.SeeFewer }.groupBy(_.engagementType) @@ -132,10 +131,9 @@ object FeedbackFatigueScorer val userDiscounts = mutable.Map[Long, Double]() feedbackEntries .collect { - case FeedbackEntry(_, _, tl.FeedbackEntity.UserId(userId), timestamp, _) => + case FeedbackEntry(_, _, tl.FeedbackEntity.UserId(userId), timestamp, _, _) => val timeSinceFeedback = queryTime.minus(timestamp) - val timeSinceDiscounting = timeSinceFeedback - DurationForFiltering - val multiplier = ((timeSinceDiscounting.inDays / ScoreMultiplierIncrementDurationInDays) + val multiplier = ((timeSinceFeedback.inDays / ScoreMultiplierIncrementDurationInDays) * ScoreMultiplierIncrement + ScoreMultiplierLowerBound) userDiscounts.update(userId, multiplier) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/NaviModelScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/NaviModelScorer.scala new file mode 100644 index 000000000..a3cc84da5 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/NaviModelScorer.scala @@ -0,0 +1,142 @@ +package com.twitter.home_mixer.functional_component.scorer + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.NaviClientConfigFeature +import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature +import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatureSet +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelIdParam +import com.twitter.home_mixer.util.NaviScorerStatsHandler +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.module.TestUserMapper +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.feature.featuremap.datarecord.AllFeatures +import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter +import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor +import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeatureMapSanitizer +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.util.Future +import com.twitter.util.Return +import java.util.UUID +import javax.inject.Inject +import javax.inject.Singleton +import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature + +object CommonFeaturesDataRecordFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +object CandidateFeaturesDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +case class NaviModelScorer @Inject() ( + predictClientFactory: PredictClientFactory, + testUserMapper: TestUserMapper, + statsReceiver: StatsReceiver) + extends Scorer[PipelineQuery, TweetCandidate] { + + override val identifier: ScorerIdentifier = ScorerIdentifier("NaviModel") + + override val features: Set[Feature[_, _]] = Set( + CommonFeaturesDataRecordFeature, + CandidateFeaturesDataRecordFeature, + PredictionRequestIdFeature, + ) ++ PredictedScoreFeatureSet.asInstanceOf[Set[Feature[_, _]]] + + private val queryDataRecordAdapter = new DataRecordConverter(AllFeatures()) + private val candidatesDataRecordAdapter = new DataRecordConverter(AllFeatures()) + + private val resultDataRecordExtractor = new DataRecordExtractor(PredictedScoreFeatureSet) + private val modelStatsHandler = new NaviScorerStatsHandler(statsReceiver, getClass.getSimpleName) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val modelId = query.params(ModelIdParam) + val naviClientConfig = + query.features.map(_.get(NaviClientConfigFeature)).get // Should always be present + + val modelStats = + modelStatsHandler.getModelStats(query) + + val modelClient = + predictClientFactory.getClient( + naviClientConfig.clientName, + naviClientConfig.customizedBatchSize) + + val predictionRequestId = UUID.randomUUID.getMostSignificantBits + val candidateAdapter = candidatesDataRecordAdapter.toDataRecord(_) + + val commonRecord = query.features.map(queryDataRecordAdapter.toDataRecord) + + def getScores( + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Future[Seq[FeatureMap]] = { + val features = candidates.map(_.features) + val records = features.map(candidateAdapter) + + val responsesFut = + modelClient.getPredictionsForBatch(records, commonRecord, modelId = Some(modelId)) + responsesFut.map { responses => + modelStats.failuresStat.add(responses.count(_.isThrow)) + modelStats.responsesStat.add(responses.size) + + if (responses.size == candidates.size) { + val predictedScoreFeatureMaps = responses.map { + case Return(dataRecord) => resultDataRecordExtractor.fromDataRecord(dataRecord) + case _ => resultDataRecordExtractor.fromDataRecord(new DataRecord()) + } + + val featureMapSanitizer = new FeatureMapSanitizer[BaseDataRecordFeature[_, _]]( + includeFeatures = PredictedScoreFeatureSet.toSet, // Ensure this is a Set[DRFeature] + statsReceiver = statsReceiver + ) + + // Sanitize the FeatureMaps + val sanitizedFeatureMaps = featureMapSanitizer.sanitize(predictedScoreFeatureMaps) + + // Add Data Record to candidate Feature Map for logging in later stages + sanitizedFeatureMaps.zip(records).map { + case (predictedScoreFeatureMap, candidateRecord) => + predictedScoreFeatureMap ++ + FeatureMapBuilder() + .add(CandidateFeaturesDataRecordFeature, candidateRecord) + .add(CommonFeaturesDataRecordFeature, commonRecord.getOrElse(new DataRecord())) + .add(PredictionRequestIdFeature, Some(predictionRequestId)) + .build() + } + } else { + modelStats.invalidResponsesCounter.incr() + throw PipelineFailure(IllegalStateFailure, "Result size mismatched candidates size") + } + } + } + + val scores = OffloadFuturePools.offloadBatchSeqToFutureSeq( + candidates, + getScores(_), + naviClientConfig.customizedBatchSize.getOrElse(predictClientFactory.DefaultRequestBatchSize), + offload = true + ) + Stitch.callFuture(scores) + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PhoenixModelRerankingScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PhoenixModelRerankingScorer.scala new file mode 100644 index 000000000..44efebd2c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PhoenixModelRerankingScorer.scala @@ -0,0 +1,81 @@ +package com.twitter.home_mixer.functional_component.scorer + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature +import com.twitter.home_mixer.model.HomeFeatures.PhoenixScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature.PhoenixPredictedScoreFeatures +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixInferenceClusterParam +import com.twitter.home_mixer.util.PhoenixScorerStatsHandler +import com.twitter.home_mixer.util.RerankerUtil._ +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class PhoenixModelRerankingScorer @Inject() (statsReceiver: StatsReceiver) + extends Scorer[PipelineQuery, TweetCandidate] { + + override val identifier: ScorerIdentifier = ScorerIdentifier("PhoenixModelReranking") + + override val features: Set[Feature[_, _]] = Set( + PhoenixScoreFeature, + // WeightedModelScoreFeature, remove temporarily to avoid overwriting navi weighted score + DebugStringFeature + ) + + private val StatsReadabilityMultiplier = 1000 + private val modelStatsHandler = + new PhoenixScorerStatsHandler(statsReceiver, getClass.getSimpleName) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val cluster = query.params(PhoenixInferenceClusterParam).toString + val modelStats = modelStatsHandler.getModelStats(query, cluster) + + val scoresAndWeightsSeq = candidates.map { candidate => + PhoenixPredictedScoreFeatures.map { feature => + val predictions = feature.extractScore(candidate.features, query) + modelStats.trackPredictedScoreStats(feature, predictions) + val isEligible = feature.isEligible(candidate.features) + val score = if (predictions.nonEmpty && isEligible) predictions.max else 0.0 + val weight = query.params(feature.modelWeightParam) + (score, weight) + } + } + val transformedScoresAndWeightsSeq = getScoresWithPerHeadMax(scoresAndWeightsSeq) + + val debugStrings: Seq[String] = + candidates.map(_.features.getOrElse(DebugStringFeature, None).getOrElse("")) + + val featureMaps = transformedScoresAndWeightsSeq + .zip(debugStrings) + .map { + case (transformedScores, debugStr) => + val finalScore = + aggregateWeightedScores(query, transformedScores, modelStats.negativeFilterCounter) + val featureNames = PhoenixPredictedScoreFeatures.map(_.featureName) + modelStats.scoreStat.add((finalScore * StatsReadabilityMultiplier).toFloat) + + val updatedDebugStr = + computeDebugMetadata(debugStr, featureNames, transformedScores, finalScore) + + FeatureMapBuilder() + .add(PhoenixScoreFeature, Some(finalScore)) + .add(DebugStringFeature, Some(updatedDebugStr)) + .build() + } + + Stitch.value(featureMaps) + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PhoenixScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PhoenixScorer.scala new file mode 100644 index 000000000..82987a924 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PhoenixScorer.scala @@ -0,0 +1,85 @@ +package com.twitter.home_mixer.functional_component.scorer + +import com.google.inject.name.Named +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.UserActionsFeature +import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature.PhoenixPredictedScoreFeatureSet +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixInferenceClusterParam +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixTimeoutInMsParam +import com.twitter.home_mixer.util.PhoenixUtils.createCandidateSets +import com.twitter.home_mixer.util.PhoenixUtils.getPredictionResponseMap +import com.twitter.home_mixer.util.PhoenixUtils.getTweetInfoFromCandidates +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.util.MemoizingStatsReceiver +import com.twitter.stitch.Stitch +import io.grpc.ManagedChannel +import javax.inject.Inject +import javax.inject.Singleton +@Singleton +case class PhoenixScorer @Inject() ( + @Named("PhoenixClient") channelsMap: Map[PhoenixCluster.Value, Seq[ManagedChannel]], + statsReceiver: StatsReceiver) + extends Scorer[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] { + + override val identifier: ScorerIdentifier = ScorerIdentifier("Phoenix") + + override val features: Set[Feature[_, _]] = + PhoenixPredictedScoreFeatureSet.asInstanceOf[Set[Feature[_, _]]] + + val memoizingStatsReceiver: MemoizingStatsReceiver = new MemoizingStatsReceiver( + statsReceiver.scope(this.getClass.getSimpleName)) + + override def onlyIf(query: PipelineQuery): Boolean = { + query.features.flatMap(_.getOrElse(UserActionsFeature, None)).isDefined + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val phoenixCluster = query.params(PhoenixInferenceClusterParam) + val channels = channelsMap(phoenixCluster) + + val tweetInfos = + getTweetInfoFromCandidates(candidates.map(_.candidate), candidates.map(_.features)) + val request = createCandidateSets(query, tweetInfos) + val timeoutMs = query.params(PhoenixTimeoutInMsParam) + val predictionsMapStitch = + getPredictionResponseMap( + request, + channels, + phoenixCluster.toString, + timeoutMs, + memoizingStatsReceiver) + + predictionsMapStitch.map { predictionsMap => + candidates.map { candidate => + val sourceTweetId = + candidate.features.getOrElse(SourceTweetIdFeature, None) match { + case Some(sourceTweetId) => sourceTweetId + case _ => candidate.candidate.id + } + val actionPredictionsMap = predictionsMap.getOrElse(sourceTweetId, Map.empty) + val fmBuilder = FeatureMapBuilder() + PhoenixPredictedScoreFeatureSet.map { feature => + val predictions = feature.actions.flatMap(actionPredictionsMap.get) + val score = if (predictions.nonEmpty) Some(predictions.max) else None + fmBuilder.add(feature, score) + } + fmBuilder.build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PredictClientFactory.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PredictClientFactory.scala new file mode 100644 index 000000000..44de4ba95 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PredictClientFactory.scala @@ -0,0 +1,83 @@ +package com.twitter.home_mixer.functional_component.scorer + +import com.twitter.finagle.stats.BroadcastStatsReceiver +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecap +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapGPU +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapRealtime +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapSecondary +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapVideo +import com.twitter.timelines.clients.predictionservice.PredictionGRPCService +import com.twitter.timelines.clients.predictionservice.PredictionServiceGRPCClient +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +case class PredictClientFactory @Inject() ( + @Named(NaviModelClientHomeRecap) homeRecapPredictionGRPCService: PredictionGRPCService, + @Named(NaviModelClientHomeRecapSecondary) homeRecapSecondaryPredictionGRPCService: PredictionGRPCService, + @Named(NaviModelClientHomeRecapRealtime) homeRecapRealtimePredictionGRPCService: PredictionGRPCService, + @Named(NaviModelClientHomeRecapGPU) homeRecapGPUPredictionGRPCService: PredictionGRPCService, + @Named(NaviModelClientHomeRecapVideo) homeRecapVideoPredictionGRPCService: PredictionGRPCService, + statsReceiver: StatsReceiver) { + + val DefaultRequestBatchSize = 32 + + def getClient( + clientName: String, + customizedBatchSize: Option[Int] + ): PredictionServiceGRPCClient = { + clientName match { + case NaviModelClientHomeRecap => + homeRecapModelClient + case NaviModelClientHomeRecapSecondary => + homeRecapSecondaryModelClient + case NaviModelClientHomeRecapRealtime => + homeRecapRealtimeModelClient + case NaviModelClientHomeRecapVideo => + homeRecapVideoModelClient + case NaviModelClientHomeRecapGPU => + val gpuBatchSize = customizedBatchSize.getOrElse(DefaultRequestBatchSize) + new PredictionServiceGRPCClient( + service = homeRecapGPUPredictionGRPCService, + statsReceiver = BroadcastStatsReceiver( + Seq(statsReceiver, statsReceiver.scope("home_recap_gpu"))), + requestBatchSize = gpuBatchSize, + useCompact = false + ) + case _ => + throw new IllegalArgumentException(s"Unknown clientName: $clientName") + } + } + + private lazy val homeRecapModelClient = new PredictionServiceGRPCClient( + service = homeRecapPredictionGRPCService, + statsReceiver = BroadcastStatsReceiver(Seq(statsReceiver, statsReceiver.scope("home_recap"))), + requestBatchSize = DefaultRequestBatchSize, + useCompact = false + ) + + private lazy val homeRecapSecondaryModelClient = new PredictionServiceGRPCClient( + service = homeRecapSecondaryPredictionGRPCService, + statsReceiver = BroadcastStatsReceiver(Seq(statsReceiver, statsReceiver.scope("home_recap"))), + requestBatchSize = DefaultRequestBatchSize, + useCompact = false + ) + + private lazy val homeRecapRealtimeModelClient = new PredictionServiceGRPCClient( + service = homeRecapRealtimePredictionGRPCService, + statsReceiver = BroadcastStatsReceiver( + Seq(statsReceiver, statsReceiver.scope("home_recap_realtime"))), + requestBatchSize = DefaultRequestBatchSize, + useCompact = false + ) + + private lazy val homeRecapVideoModelClient = new PredictionServiceGRPCClient( + service = homeRecapVideoPredictionGRPCService, + statsReceiver = BroadcastStatsReceiver( + Seq(statsReceiver, statsReceiver.scope("home_recap_video"))), + requestBatchSize = DefaultRequestBatchSize, + useCompact = false + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/WeighedModelRerankingScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/WeighedModelRerankingScorer.scala new file mode 100644 index 000000000..99eb72b72 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/WeighedModelRerankingScorer.scala @@ -0,0 +1,71 @@ +package com.twitter.home_mixer.functional_component.scorer + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature +import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatures +import com.twitter.home_mixer.util.NaviScorerStatsHandler +import com.twitter.home_mixer.util.RerankerUtil._ +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class WeighedModelRerankingScorer @Inject() ( + statsReceiver: StatsReceiver) + extends Scorer[PipelineQuery, TweetCandidate] { + + override val identifier: ScorerIdentifier = ScorerIdentifier("WeightedModelReranking") + + override val features: Set[Feature[_, _]] = Set( + ScoreFeature, + WeightedModelScoreFeature, + DebugStringFeature + ) + + private val modelStatsHandler = new NaviScorerStatsHandler(statsReceiver, getClass.getSimpleName) + private val StatsReadabilityMultiplier = 1000 + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val modelStats = modelStatsHandler.getModelStats(query) + val scoresAndWeightsSeq = candidates.map(computeModelScores(query, _, Some(modelStats))) + val transformedScoresAndWeightsSeq = getScoresWithPerHeadMax(scoresAndWeightsSeq) + + val debugStrings: Seq[String] = + candidates.map(_.features.getOrElse(DebugStringFeature, None).getOrElse("")) + + val featureMaps = transformedScoresAndWeightsSeq + .zip(debugStrings) + .map { + case (transformedScores, debugStr) => + val finalScore = + aggregateWeightedScores(query, transformedScores, modelStats.negativeFilterCounter) + val featureNames = PredictedScoreFeatures.map(_.statName) + modelStats.scoreStat.add((finalScore * StatsReadabilityMultiplier).toFloat) + + val updatedDebugStr = + computeDebugMetadata(debugStr, featureNames, transformedScores, finalScore) + + FeatureMapBuilder() + .add(ScoreFeature, Some(finalScore)) + .add(WeightedModelScoreFeature, Some(finalScore)) + .add(DebugStringFeature, Some(updatedDebugStr)) + .build() + } + + Stitch.value(featureMaps) + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel index 3689de6ea..bd6af7549 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel @@ -4,22 +4,17 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "src/scala/com/twitter/suggests/controller_data", "stringcenter/client", - "stringcenter/client/src/main/java", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/RandomShuffleCandidates.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/RandomShuffleCandidates.scala new file mode 100644 index 000000000..a0e1003e2 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/RandomShuffleCandidates.scala @@ -0,0 +1,23 @@ +package com.twitter.home_mixer.functional_component.selector + +import com.twitter.product_mixer.core.functional_component.common.AllPipelines +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import scala.util.Random + +object RandomShuffleCandidates extends Selector[PipelineQuery] { + + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val shuffledRemainingCandidates = Random.shuffle(remainingCandidates) + SelectorResult(remainingCandidates = shuffledRemainingCandidates, result = result) + } + + override def pipelineScope: CandidateScope = AllPipelines +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/ScoreAveragingPositionSelector.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/ScoreAveragingPositionSelector.scala new file mode 100644 index 000000000..6035ec4e1 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/ScoreAveragingPositionSelector.scala @@ -0,0 +1,238 @@ +package com.twitter.home_mixer.functional_component.selector + +import com.twitter.home_mixer.model.HomeFeatures.IsBoostedCandidateFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.UserFollowersCountFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CategoryColdStartProbabilisticReturnParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CategoryColdStartTierOneProbabilityParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ContentExplorationBoostPosParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ContentExplorationViewerMaxFollowersParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.DeepRetrievalBoostPosParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.DeepRetrievalI2iProbabilityParam +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.common.AllPipelines +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import scala.util.Random + +/** + * Abstract base class specifically for fixed position selectors that need score averaging + * Contains complete selector logic, supports removing target candidates from remaining candidates + * and assigning them the average score of adjacent candidates + */ +abstract class ScoreAveragingPositionSelector extends Selector[PipelineQuery] { + + // Abstract methods that subclasses need to implement + def getOffset(query: PipelineQuery): Int + def getNumCandidatesToBoost(query: PipelineQuery): Int + def selectEligibleCandidates(query: PipelineQuery, candidate: CandidateWithDetails): Boolean + + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val eligibleCandidates = remainingCandidates.filter(c => selectEligibleCandidates(query, c)) + val targetCandidates = eligibleCandidates.take(getNumCandidatesToBoost(query)) + + if (targetCandidates.nonEmpty) { + val filteredRemainingCandidates = remainingCandidates.diff(targetCandidates) + val offset = getOffset(query) + + // Get scores from adjacent positions for average calculation + val adjacentScores = getAdjacentScores(filteredRemainingCandidates, offset) + + // Assign target candidates scores based on the average of adjacent candidates + val scoredTargetCandidates = + targetCandidates.map(candidate => assignAverageScore(candidate, adjacentScores)) + + val updatedRemainingCandidates = + if (offset >= 0 && offset <= filteredRemainingCandidates.length) { + filteredRemainingCandidates.slice(0, offset) ++ scoredTargetCandidates ++ + filteredRemainingCandidates.slice(offset, filteredRemainingCandidates.length) + } else { + filteredRemainingCandidates ++ scoredTargetCandidates + } + + SelectorResult(remainingCandidates = updatedRemainingCandidates, result = result) + } else { + SelectorResult(remainingCandidates = remainingCandidates, result = result) + } + } + + // Get adjacent position scores + def getAdjacentScores( + filteredRemainingCandidates: Seq[CandidateWithDetails], + offset: Int + ): Seq[Option[Double]] = { + if (offset >= 1 && offset < filteredRemainingCandidates.length) { + Seq( + filteredRemainingCandidates(offset - 1).features.getOrElse(ScoreFeature, None), + filteredRemainingCandidates(offset).features.getOrElse(ScoreFeature, None) + ) + } else if (offset == 0 && filteredRemainingCandidates.nonEmpty) { + Seq(filteredRemainingCandidates.head.features.getOrElse(ScoreFeature, None)) + } else if (offset >= filteredRemainingCandidates.length && filteredRemainingCandidates.nonEmpty) { + Seq(filteredRemainingCandidates.last.features.getOrElse(ScoreFeature, None)) + } else { + Seq.empty + } + } + + // Assign scores to candidates based on the average score of adjacent candidates + def assignAverageScore( + candidate: CandidateWithDetails, + adjacentScores: Seq[Option[Double]] + ): CandidateWithDetails = { + val validScores = adjacentScores.flatten + val avgScore = if (validScores.nonEmpty) { + Some(validScores.sum / validScores.size) + } else None + + candidate match { + case item: ItemCandidateWithDetails if avgScore.isDefined => + val updatedFeatures = FeatureMapBuilder() + .add(ScoreFeature, avgScore) + .add(IsBoostedCandidateFeature, true) + .build() + item.copy(features = item.features ++ updatedFeatures) + case _ => candidate + } + } + + def pipelineScope: CandidateScope = AllPipelines +} + +/** + * Concrete implementations of score averaging position selectors + */ +object SortFixedPositionContentExplorationSimclusterColdPostsCandidates + extends ScoreAveragingPositionSelector { + + override def getOffset(query: PipelineQuery): Int = + query.params.getInt(ContentExplorationBoostPosParam) + + override def getNumCandidatesToBoost(query: PipelineQuery): Int = 1 + + override def selectEligibleCandidates( + query: PipelineQuery, + candidate: CandidateWithDetails + ): Boolean = { + val servedType = candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) + servedType == hmt.ServedType.ForYouContentExplorationSimclusterColdPosts + } +} + +object SortFixedPositionContentExplorationMixedCandidates extends ScoreAveragingPositionSelector { + + override def getOffset(query: PipelineQuery): Int = + query.params.getInt(ContentExplorationBoostPosParam) + + override def getNumCandidatesToBoost(query: PipelineQuery): Int = 1 + + override def selectEligibleCandidates( + query: PipelineQuery, + candidate: CandidateWithDetails + ): Boolean = true + + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val chosenServedType = + if (Random.nextDouble() < query.params.getDouble(CategoryColdStartTierOneProbabilityParam)) + hmt.ServedType.ForYouContentExploration + else hmt.ServedType.ForYouContentExplorationTier2 + + val eligibleCandidates = remainingCandidates + .filter(_.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) == chosenServedType) + + val viewerMaxFollowers = query.params(ContentExplorationViewerMaxFollowersParam) + val viewerFollowers = query.features.flatMap(_.getOrElse(UserFollowersCountFeature, None)) + + val returnCandidatesRandomDraw = + Random.nextDouble() < query.params.getDouble(CategoryColdStartProbabilisticReturnParam) + + if (viewerFollowers.forall(_ < viewerMaxFollowers) && + eligibleCandidates.nonEmpty && returnCandidatesRandomDraw) { + val targetCandidates = eligibleCandidates.take(getNumCandidatesToBoost(query)) + + val filteredRemainingCandidates = remainingCandidates.diff(targetCandidates) + val offset = getOffset(query) + + // Get scores from adjacent positions for average calculation + val adjacentScores = getAdjacentScores(filteredRemainingCandidates, offset) + + // Assign target candidates scores based on the average of adjacent candidates + val scoredTargetCandidates = + targetCandidates.map(candidate => assignAverageScore(candidate, adjacentScores)) + + val updatedRemainingCandidates = + if (offset >= 0 && offset <= filteredRemainingCandidates.length) { + filteredRemainingCandidates.slice(0, offset) ++ scoredTargetCandidates ++ + filteredRemainingCandidates.slice(offset, filteredRemainingCandidates.length) + } else filteredRemainingCandidates ++ scoredTargetCandidates + + SelectorResult(remainingCandidates = updatedRemainingCandidates, result = result) + } else SelectorResult(remainingCandidates = remainingCandidates, result = result) + } +} + +object SortFixedPositionDeepRetrievalMixedCandidates extends ScoreAveragingPositionSelector { + + override def getOffset(query: PipelineQuery): Int = + query.params.getInt(DeepRetrievalBoostPosParam) + + override def getNumCandidatesToBoost(query: PipelineQuery): Int = 1 + + override def selectEligibleCandidates( + query: PipelineQuery, + candidate: CandidateWithDetails + ): Boolean = true + + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val chosenServedType = + if (Random.nextDouble() < query.params.getDouble(DeepRetrievalI2iProbabilityParam)) + hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i + else hmt.ServedType.ForYouContentExplorationTier2DeepRetrievalI2i + + val eligibleCandidates = remainingCandidates + .filter(_.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) == chosenServedType) + + val viewerMaxFollowers = query.params(ContentExplorationViewerMaxFollowersParam) + val viewerFollowers = query.features.flatMap(_.getOrElse(UserFollowersCountFeature, None)) + + if (viewerFollowers.forall(_ < viewerMaxFollowers) && eligibleCandidates.nonEmpty) { + val targetCandidates = Seq(eligibleCandidates(Random.nextInt(eligibleCandidates.size))) + + val filteredRemainingCandidates = remainingCandidates.diff(targetCandidates) + val offset = getOffset(query) + + // Get scores from adjacent positions for average calculation + val adjacentScores = getAdjacentScores(filteredRemainingCandidates, offset) + + // Assign target candidates scores based on the average of adjacent candidates + val scoredTargetCandidates = + targetCandidates.map(candidate => assignAverageScore(candidate, adjacentScores)) + + val updatedRemainingCandidates = + if (offset >= 0 && offset <= filteredRemainingCandidates.length) { + filteredRemainingCandidates.slice(0, offset) ++ scoredTargetCandidates ++ + filteredRemainingCandidates.slice(offset, filteredRemainingCandidates.length) + } else filteredRemainingCandidates ++ scoredTargetCandidates + + SelectorResult(remainingCandidates = updatedRemainingCandidates, result = result) + } else SelectorResult(remainingCandidates = remainingCandidates, result = result) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/SortFixedPositionCandidates.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/SortFixedPositionCandidates.scala new file mode 100644 index 000000000..eb35f99fb --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/SortFixedPositionCandidates.scala @@ -0,0 +1,57 @@ +package com.twitter.home_mixer.functional_component.selector + +import com.twitter.product_mixer.core.functional_component.common.AllPipelines +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +abstract class SortFixedPositionCandidates extends Selector[PipelineQuery] { + + def getOffset(query: PipelineQuery): Int + + def getNumCandidatesToBoost(query: PipelineQuery): Int + + def selectEligibleCandidates( + query: PipelineQuery, + candidate: CandidateWithDetails + ): Boolean + + def selectTargetFromRemainingCandidates( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails] + ): Seq[CandidateWithDetails] = { + val numCandidatesToBoost = getNumCandidatesToBoost(query) + val eligibleCandidates = remainingCandidates + .filter(candidate => selectEligibleCandidates(query, candidate)) + + eligibleCandidates.take(numCandidatesToBoost) + } + + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val targetCandidates = selectTargetFromRemainingCandidates(query, remainingCandidates) + val offset = getOffset(query) + if (targetCandidates.nonEmpty) { + val updatedRemainingCandidates = if (offset >= 0 && offset < remainingCandidates.length) { + remainingCandidates.slice(0, offset) ++ targetCandidates ++ remainingCandidates.slice( + offset, + remainingCandidates.length) + } else { + remainingCandidates ++ targetCandidates + } + SelectorResult( + remainingCandidates = updatedRemainingCandidates.distinct, + result = result + ) + } else { + SelectorResult(remainingCandidates = remainingCandidates, result = result) + } + } + + override def pipelineScope: CandidateScope = AllPipelines +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala index c7944b506..a6089933b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala @@ -2,14 +2,18 @@ package com.twitter.home_mixer.functional_component.selector import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventDetailsBuilder import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature -import com.twitter.home_mixer.model.HomeFeatures.HasRandomTweetFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetAboveFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokCategoryDataRecordFeature +import com.twitter.home_mixer.model.HomeFeatures.MaxSingleAuthorCountFeature +import com.twitter.home_mixer.model.HomeFeatures.MaxSingleCategoryCountFeature import com.twitter.home_mixer.model.HomeFeatures.PositionFeature import com.twitter.home_mixer.model.HomeFeatures.ServedInConversationModuleFeature import com.twitter.home_mixer.model.HomeFeatures.ServedSizeFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.UniqueAuthorCountFeature +import com.twitter.home_mixer.model.HomeFeatures.UniqueCategoryCountFeature import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -52,13 +56,26 @@ case class UpdateHomeClientEventDetails(candidatePipelines: Set[CandidatePipelin ): SelectorResult = { val selectedCandidates = result.filter(pipelineScope.contains) - val randomTweetsByPosition = result - .map(_.features.getOrElse(IsRandomTweetFeature, false)) - .zipWithIndex.map(_.swap).toMap + val authorCounts: Map[Long, Int] = selectedCandidates + .flatMap(_.features.getOrElse(AuthorIdFeature, None)) + .groupBy(identity) + .map { case (authorId, ids) => authorId -> ids.length } + + val categoryCounts: Map[String, Int] = selectedCandidates + .flatMap(_.features.getOrElse(GrokCategoryDataRecordFeature, None).getOrElse(Map.empty).keys) + .groupBy(identity) + .map { case (category, categories) => category -> categories.length } val resultFeatures = FeatureMapBuilder() .add(ServedSizeFeature, Some(selectedCandidates.size)) - .add(HasRandomTweetFeature, randomTweetsByPosition.valuesIterator.contains(true)) + .add(UniqueAuthorCountFeature, Some(authorCounts.size)) + .add( + MaxSingleAuthorCountFeature, + Some(if (authorCounts.values.nonEmpty) authorCounts.values.max else 0)) + .add(UniqueCategoryCountFeature, Some(categoryCounts.size)) + .add( + MaxSingleCategoryCountFeature, + Some(if (categoryCounts.values.nonEmpty) categoryCounts.values.max else 0)) .build() val updatedResult = result.zipWithIndex.map { @@ -66,7 +83,6 @@ case class UpdateHomeClientEventDetails(candidatePipelines: Set[CandidatePipelin if pipelineScope.contains(item) => val resultCandidateFeatures = FeatureMapBuilder() .add(PositionFeature, Some(position)) - .add(IsRandomTweetAboveFeature, randomTweetsByPosition.getOrElse(position - 1, false)) .build() updateItemPresentation(query, item, resultFeatures, resultCandidateFeatures) @@ -75,12 +91,12 @@ case class UpdateHomeClientEventDetails(candidatePipelines: Set[CandidatePipelin if pipelineScope.contains(module) => val resultCandidateFeatures = FeatureMapBuilder() .add(PositionFeature, Some(position)) - .add(IsRandomTweetAboveFeature, randomTweetsByPosition.getOrElse(position - 1, false)) .add(ServedInConversationModuleFeature, true) .add(ConversationModule2DisplayedTweetsFeature, module.candidates.size == 2) .add( ConversationModuleHasGapFeature, module.candidates.last.features.getOrElse(AncestorsFeature, Seq.empty).size > 2) + .add(ServedTypeFeature, module.candidates.last.features.get(ServedTypeFeature)) .build() val updatedItemCandidates = diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel index f352ce63a..c64d5acb3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel @@ -1,47 +1,54 @@ scala_library( - sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, - tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", + "3rdparty/jvm/com/twitter/storehaus:core", "eventbus/client/src/main/scala/com/twitter/eventbus/client", + "finagle/finagle-mysql/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweet_mixer", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/communities_to_join", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/job", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/recruiting_organization", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", + "src/scala/com/twitter/suggests/controller_data", + "src/scala/com/twitter/timelines/prediction/adapters/twistly", + "src/scala/com/twitter/timelines/prediction/adapters/two_hop_features", + "src/scala/com/twitter/timelines/prediction/features/large_embeddings", "src/thrift/com/twitter/timelines/impression_store:thrift-scala", "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", + "src/thrift/com/twitter/timelines/suggests/common:data_record_metadata-scala", "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", - "src/thrift/com/twitter/user_session_store:thrift-scala", + "strato/config/columns/videoRecommendations/twitterClip:twitterClip-strato-client", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelines/ml:kafka", - "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka", + "timelines/ml:pldr-client", + "timelines/ml:pldr-conversion", "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", "timelines/src/main/scala/com/twitter/timelines/clientconfig", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/store", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", + "timelines/src/main/scala/com/twitter/timelines/util/stats", "timelineservice/common:model", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", "user_session_store/src/main/scala/com/twitter/user_session_store", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BaseCacheCandidateFeaturesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BaseCacheCandidateFeaturesSideEffect.scala new file mode 100644 index 000000000..fba8cce09 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BaseCacheCandidateFeaturesSideEffect.scala @@ -0,0 +1,468 @@ +package com.twitter.home_mixer.functional_component.side_effect + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.filter.OffloadFilter +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.mysql.Client +import com.twitter.finagle.mysql.Transactions +import com.twitter.finagle.offload.OffloadFuturePool +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.transport.Transport +import com.twitter.finagle.util.DefaultTimer +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.ClipEmbeddingFeaturesAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.TextTokensFeaturesAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.light_ranking_features.LightRankingCandidateFeatures +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.light_ranking_features.LightRankingCandidateFeaturesAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeatures +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeaturesAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCommonFeatures +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCommonFeaturesAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinVideoEmbeddingsAdapter +import com.twitter.home_mixer.functional_component.scorer.CandidateFeaturesDataRecordFeature +import com.twitter.home_mixer.model.HomeFeatures.ClientIdFeature +import com.twitter.home_mixer.model.HomeFeatures.GuestIdFeature +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature +import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetTextTokensFeature +import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature +import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatureSet +import com.twitter.home_mixer.model.request.FollowingProduct +import com.twitter.home_mixer.model.request.ForYouProduct +import com.twitter.home_mixer.model.request.ScoredTweetsProduct +import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct +import com.twitter.home_mixer.model.request.SubscribedProduct +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableClipEmbeddingFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetTextTokensEmbeddingFeatureScribingParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetVideoAggregatedWatchTimeFeatureScribingParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinVideoFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableVideoClipEmbeddingFeatureHydrationDeciderParam +import com.twitter.home_mixer.param.HomeGlobalParams.IsSelectedByHeavyRankerCountParam +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer.util.CandidatesUtil.getOriginalAuthorId +import com.twitter.home_mixer.util.CandidatesUtil.getOriginalTweetId +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.DataRecordMerger +import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter +import com.twitter.product_mixer.core.feature.featuremap.datarecord.SpecificFeatures +import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateSourcePosition +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.HasMarshalling +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.storehaus.Store +import com.twitter.strato.generated.client.videoRecommendations.twitterClip.TwitterClipEmbeddingMhClientColumn +import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.CandidateAndCommonFeaturesStreamingUtils +import com.twitter.timelines.ml.pldr.client.MysqlClientUtils +import com.twitter.timelines.ml.pldr.client.VersionedMetadataCacheClient +import com.twitter.timelines.ml.pldr.conversion.VersionIdAndFeatures +import com.twitter.timelines.prediction.adapters.twistly.VideoAggregatedWatchTimeFeaturesAdapter +import com.twitter.timelines.prediction.features.large_embeddings.LargeEmbeddingsFeatures.AllCandidateLargeEmbeddingsFeatures +import com.twitter.timelines.served_candidates_logging.{thriftscala => sc} +import com.twitter.timelines.suggests.common.data_record_metadata.{thriftscala => drmd} +import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} +import com.twitter.timelines.util.stats.FutureObserver +import com.twitter.timelines.util.stats.OptionObserver +import com.twitter.twistly.thriftscala.VideoViewEngagementType +import com.twitter.twistly.thriftscala.WatchTimeMetadata +import com.twitter.twistly.{thriftscala => ts} +import com.twitter.util.Future +import com.twitter.util.Try +import com.twitter.util.logging.Logging +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + +@Singleton +class BaseCacheCandidateFeaturesSideEffect @Inject() ( + dataRecordMetadataStoreConfigsYml: String, + store: Store[ + sc.CandidateFeatureKey, + pldr.PolyDataRecord + ], + tweetWatchTimeMetadataStore: ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata], + twitterClipEmbeddingMhClientColumn: TwitterClipEmbeddingMhClientColumn, + twhinVideoStore: ReadableStore[Long, TwhinTweetEmbedding], + statsReceiver: StatsReceiver) + extends PipelineResultSideEffect[PipelineQuery, HasMarshalling] + with Conditionally[PipelineQuery, HasMarshalling] + with Logging { + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Boolean = { + val serviceIdentifier = ServiceIdentifier.fromCertificate(Transport.peerCertificate) + (selectedCandidates.nonEmpty || remainingCandidates.nonEmpty || droppedCandidates.nonEmpty) && serviceIdentifier.role != "video-mixer" + } + + override val alerts: Seq[SuccessRateAlert] = Seq( + HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5)) + + val identifier: SideEffectIdentifier = + SideEffectIdentifier("BaseCacheCandidateFeaturesSideEffect") + val statScope: String = this.getClass.getSimpleName + + private val scopedStatsReceiver = statsReceiver.scope(statScope) + private val metadataFetchFailedCounter = scopedStatsReceiver.counter("metadataFetchFailed") + private val writesRequestCounter = scopedStatsReceiver.counter("writesRequests") + private val writesFailedCounter = scopedStatsReceiver.counter("writesFailed") + + private val twhinVideoEmbeddingFutureObserver = FutureObserver( + scopedStatsReceiver.scope("twhinVideoEmbedding")) + private val twhinVideoEmbeddingOptionObserver = OptionObserver( + scopedStatsReceiver.scope("twhinVideoEmbedding")) + + private val clipEmbeddingCounter = scopedStatsReceiver.counter("clipEmbeddingCounter") + + private val tweetWatchTimeMetadataRequestCounter = + scopedStatsReceiver.counter("tweetWatchTimeMetadataRequests") + private val tweetWatchTimeMetadataSuccessCounter = + scopedStatsReceiver.counter("tweetWatchTimeMetadataSuccessCounter") + private val tweetWatchTimeMetadataFailureCounter = + scopedStatsReceiver.counter("tweetWatchTimeMetadataFailureCounter") + + private val drMerger = new DataRecordMerger + private val predictedScoreFeaturesDataRecordAdapter = + new DataRecordConverter(SpecificFeatures(PredictedScoreFeatureSet)) + + lazy private val dataRecordMetadataStoreClient: Option[Client with Transactions] = Try { + try { + val c = MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml) + logger.info(s"pldr mysql config: ${c.host} ${c.port} ${c.user} ${c.database}") + } catch { + case e: Throwable => + logger.error("pldr mysql error: " + e.toString) + } + + MysqlClientUtils.mysqlClientProvider( + MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml) + ) + }.toOption + + lazy private val versionedMetadataCacheClientOpt: Option[ + VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]] + ] = dataRecordMetadataStoreClient.map { mysqlClient => + new VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]]( + maximumSize = 1, + expireDurationOpt = None, + mysqlClient = mysqlClient, + transform = CandidateAndCommonFeaturesStreamingUtils.metadataTransformer, + statsReceiver = statsReceiver + ) + } + + versionedMetadataCacheClientOpt.foreach { + _.metadataFetchTimerTask( + CandidateAndCommonFeaturesStreamingUtils.metadataFetchKey, + metadataFetchTimer = DefaultTimer, + metadataFetchInterval = 90.seconds, + metadataFetchFailedCounter = metadataFetchFailedCounter + ) + } + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, HasMarshalling] + ): Stitch[Unit] = Stitch.let(OffloadFuturePool.lowPriorityLocal)(()) { + OffloadFuturePools.offloadStitch { + + // Exclude unscored candidates (e.g. because of scoring QF truncation or cache reads) + val selectedCandidates = + (inputs.selectedCandidates ++ inputs.remainingCandidates) + .filter { candidate => + candidate.features.contains(CandidateFeaturesDataRecordFeature) + } + val droppedCandidates = + inputs.droppedCandidates + .filter { candidate => + candidate.features.contains(CandidateFeaturesDataRecordFeature) + } + + val candidates = (selectedCandidates ++ droppedCandidates) + val isSelectedCandidateIds: Set[Long] = selectedCandidates.map(_.candidateIdLong).toSet + val candidatesHeavyRankerScoreBasedRank: Map[Long, Int] = candidates + .sortBy(-_.features + .getOrElse(WeightedModelScoreFeature, None).getOrElse(Double.NegativeInfinity)).map( + _.candidateIdLong).zipWithIndex.toMap + val isSelectedByHeavyRankerCount = inputs.query.params(IsSelectedByHeavyRankerCountParam) + + val predictionRequestId = candidates.headOption.flatMap { candidate => + candidate.features.getOrElse(PredictionRequestIdFeature, None) + } + + val productSurface = inputs.query.product match { + case FollowingProduct => hmt.Product.Following + case ForYouProduct => hmt.Product.ForYou + case ScoredTweetsProduct => hmt.Product.ScoredTweets + case ScoredVideoTweetsProduct => hmt.Product.ScoredVideoTweets + case SubscribedProduct => hmt.Product.Subscribed + case other => throw new UnsupportedOperationException(s"Unknown product: $other") + } + + val nonMLCommonFeatures = NonMLCommonFeatures( + userId = inputs.query.getRequiredUserId, + guestId = inputs.query.features.flatMap(_.getOrElse(GuestIdFeature, None)), + clientId = inputs.query.features.flatMap(_.getOrElse(ClientIdFeature, None)), + countryCode = inputs.query.getCountryCode, + predictionRequestId = predictionRequestId, + productSurface = productSurface.toString, + servedTimestamp = inputs.query.queryTime.inMilliseconds + ) + + val nonMLCommonFeaturesDataRecord = + NonMLCommonFeaturesAdapter.adaptToDataRecords(nonMLCommonFeatures).asScala.head + + val sideEffectStitch = fetchExperimentalFeatures(inputs.query, candidates) + .map { candidatesAndFeatures => + // Further writes are cheap and do not have callbacks, + // therefore it's safe to bypass offload filter. + OffloadFilter.withOffloadsDisabled { + candidatesAndFeatures.foreach { + case (candidate, experimentalFeatures) => + val candidateFeaturesPldr = buildCandidateFeaturesPldr( + query = inputs.query, + candidate = candidate, + isSelected = isSelectedCandidateIds.contains(candidate.candidateIdLong), + isSelectedByHeavyRanker = candidatesHeavyRankerScoreBasedRank + .getOrElse( + candidate.candidateIdLong, + candidates.size) < isSelectedByHeavyRankerCount, + rankByHeavyRanker = candidatesHeavyRankerScoreBasedRank + .getOrElse(candidate.candidateIdLong, candidates.size), + nonMLCommonFeaturesDataRecord = nonMLCommonFeaturesDataRecord, + experimentalFeaturesDataRecords = experimentalFeatures, + ) + + writesRequestCounter.incr() + val candidateFeaturesKey = sc.CandidateFeatureKey( + tweetId = candidate.candidateIdLong, + viewerId = inputs.query.getRequiredUserId, + servedId = predictionRequestId.getOrElse(-1L) + ) + + store.put(candidateFeaturesKey -> candidateFeaturesPldr).rescue { + case _: Throwable => + writesFailedCounter.incr() + Future.Unit + } + } + } + } + + Stitch.run(sideEffectStitch) + + Stitch.Unit + } + } + + private def buildCandidateFeaturesPldr( + query: PipelineQuery, + candidate: CandidateWithDetails, + isSelected: Boolean, + isSelectedByHeavyRanker: Boolean, + rankByHeavyRanker: Int, + nonMLCommonFeaturesDataRecord: DataRecord, + experimentalFeaturesDataRecords: Seq[DataRecord] + ): Option[pldr.PolyDataRecord] = { + // Step 1) Set candidate features to all existing candidate features used in ranking + val candidateFeaturesDataRecord = + candidate.features.get(CandidateFeaturesDataRecordFeature) + + // Step 2) Remove all large embeddings features from DataRecord + AllCandidateLargeEmbeddingsFeatures.foreach { feature => + candidateFeaturesDataRecord.tensors.remove(feature.getFeatureId) + } + + // Step 3) Add prediction score + val predictedScoreFeaturesDataRecord = + predictedScoreFeaturesDataRecordAdapter.toDataRecord(candidate.features) + drMerger.merge(candidateFeaturesDataRecord, predictedScoreFeaturesDataRecord) + + // Step 4) Add non-ML common features + drMerger.merge(candidateFeaturesDataRecord, nonMLCommonFeaturesDataRecord) + + // Step 5) Add non-ML candidate features, including light ranking features + val nonMLCandidateFeatures = NonMLCandidateFeatures( + tweetId = candidate.candidateIdLong, + sourceTweetId = getOriginalTweetId(candidate.candidateIdLong, candidate.features), + originalAuthorId = getOriginalAuthorId(candidate.features) + ) + val nonMLCandidateFeaturesDataRecord = + NonMLCandidateFeaturesAdapter.adaptToDataRecords(nonMLCandidateFeatures).asScala.head + val lightRankingCandidateFeatures = LightRankingCandidateFeatures( + isSelected = isSelected, + isSelectedByHeavyRanker = isSelectedByHeavyRanker, + rankByHeavyRanker = rankByHeavyRanker, + servedType = candidate.features.get(ServedTypeFeature), + candidateSourcePosition = candidate.features.get(CandidateSourcePosition).toLong + ) + val lightRankingCandidateFeaturesDataRecord = + LightRankingCandidateFeaturesAdapter + .adaptToDataRecords(lightRankingCandidateFeatures).asScala.head + drMerger.merge(nonMLCandidateFeaturesDataRecord, lightRankingCandidateFeaturesDataRecord) + drMerger.merge(candidateFeaturesDataRecord, nonMLCandidateFeaturesDataRecord) + + // Step 5) Add experimental features (including twhin) + experimentalFeaturesDataRecords.foreach(drMerger.merge(candidateFeaturesDataRecord, _)) + + CandidateAndCommonFeaturesStreamingUtils.candidateFeaturesToPolyDataRecord( + versionedMetadataCacheClientOpt = versionedMetadataCacheClientOpt, + candidateFeatures = candidateFeaturesDataRecord, + valueFormat = pldr.PolyDataRecord._Fields.LITE_COMPACT_DATA_RECORD + ) + } + + private def fetchExperimentalFeatures( + query: PipelineQuery, + candidates: Seq[CandidateWithDetails], + ): Stitch[Map[CandidateWithDetails, Seq[DataRecord]]] = { + val stitches = Seq( + fetchTwhinVideoEmbeddingDataRecords(query, candidates), + fetchTweetTextTokensDataRecord(query, candidates), + fetchClipEmbeddings(query, candidates), + fetchTweetVideoAggregatedWatchTimeDataRecord(query, candidates), + ) + Stitch + .collect(stitches) + .map { candidatesMaps => + val candidatesToDataRecords = + new java.util.HashMap[CandidateWithDetails, ArrayBuffer[DataRecord]]( + candidates.size * 3 / 4) + candidatesMaps.foreach { candidatesToDataRecord => + candidatesToDataRecord.foreach { + case (candidate, dataRecord) => + if (dataRecord.isDefined) { + candidatesToDataRecords + .computeIfAbsent( + candidate, + _ => new ArrayBuffer[DataRecord](stitches.size) + ).append(dataRecord.get) + } + } + } + candidatesToDataRecords.asScala.toMap + } + } + + private def fetchTwhinVideoEmbeddingDataRecords( + query: PipelineQuery, + candidates: Seq[CandidateWithDetails], + ): Stitch[Map[CandidateWithDetails, Option[DataRecord]]] = { + if (!query.params(EnableTwhinVideoFeaturesParam)) { + Stitch.value(Map.empty) + } else { + val originalTweetToCandidates = candidates.groupBy { CandidatesUtil.getOriginalTweetId(_) } + Stitch.callFuture { + twhinVideoEmbeddingFutureObserver( + Future.collect(twhinVideoStore.multiGet(originalTweetToCandidates.keySet)).map { + originalTweetIdToEmbeddings => + originalTweetIdToEmbeddings.flatMap { + case (originalTweetId, embeddingOpt) => + val floatTensor = twhinVideoEmbeddingOptionObserver(embeddingOpt) + .map { e => ml.FloatTensor(e.embedding) } + val dataRecord = + TwhinVideoEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.headOption + originalTweetToCandidates(originalTweetId).map { candidate => + candidate -> dataRecord + } + } + } + ) + } + } + } + + private def fetchClipEmbeddings( + query: PipelineQuery, + candidates: Seq[CandidateWithDetails], + ): Stitch[Map[CandidateWithDetails, Option[DataRecord]]] = { + Stitch + .collect( + candidates.map { candidate => + if (query.params(EnableClipEmbeddingFeaturesParam) && + candidate.features.getOrElse(HasVideoFeature, false) && + !query.params( + EnableVideoClipEmbeddingFeatureHydrationDeciderParam + )) { // If it's not enabled through feature hydrator + clipEmbeddingCounter.incr() + twitterClipEmbeddingMhClientColumn.fetcher + .fetch(candidate.candidateIdLong) + .map { result => + candidate -> result.v.flatMap { record => + ClipEmbeddingFeaturesAdapter + .adaptToDataRecords(record) + .asScala + .headOption + } + } + } else { + Stitch.value(candidate -> None) + } + } + ).map(_.toMap) + } + + private def fetchTweetTextTokensDataRecord( + query: PipelineQuery, + candidates: Seq[CandidateWithDetails], + ): Stitch[Map[CandidateWithDetails, Option[DataRecord]]] = { + Stitch.value { + if (query.params(EnableTweetTextTokensEmbeddingFeatureScribingParam)) { + Map.empty + } else { + candidates.map { candidate => + candidate -> candidate.features.getOrElse(TweetTextTokensFeature, None).flatMap { + textTokens => + TextTokensFeaturesAdapter + .adaptToDataRecords(textTokens) + .asScala + .headOption + } + }.toMap + } + } + } + private def fetchTweetVideoAggregatedWatchTimeDataRecord( + query: PipelineQuery, + candidates: Seq[CandidateWithDetails], + ): Stitch[Map[CandidateWithDetails, Option[DataRecord]]] = { + Stitch + .collect( + candidates.map { candidate => + if (query.params( + EnableTweetVideoAggregatedWatchTimeFeatureScribingParam) && candidate.features + .getOrElse(HasVideoFeature, false)) { + tweetWatchTimeMetadataRequestCounter.incr() + Stitch.callFuture( + tweetWatchTimeMetadataStore + .get( + (candidate.candidateIdLong, ts.VideoViewEngagementType.ImmersiveVideoWatchTime) + ).onSuccess { _ => tweetWatchTimeMetadataSuccessCounter.incr() } + .onFailure { _ => tweetWatchTimeMetadataFailureCounter.incr() } + .map { opt => + candidate -> opt.flatMap(VideoAggregatedWatchTimeFeaturesAdapter + .adaptToDataRecords(_).asScala.headOption) + } + ) + } else { + Stitch.value(candidate -> None) + } + } + ).map(_.toMap) + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala index a5cf739d3..2c75a54bd 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala @@ -4,11 +4,11 @@ import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetTypeMetricsFeature import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace @@ -16,19 +16,17 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.injection.scribe.InjectionScribeUtil +import com.twitter.suggests.controller_data.Home private[side_effect] sealed trait ClientEventsBuilder { private val FollowingSection = Some("latest") private val ForYouSection = Some("home") - private val ListTweetsSection = Some("list") private val SubscribedSection = Some("subscribed") protected def section(query: PipelineQuery): Option[String] = { query.product match { case FollowingProduct => FollowingSection case ForYouProduct => ForYouSection - case ListTweetsProduct => ListTweetsSection case SubscribedProduct => SubscribedSection case other => throw new UnsupportedOperationException(s"Unknown product: $other") } @@ -52,20 +50,41 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder { private val ServedTweetsAction = Some("served_tweets") private val ServedUsersAction = Some("served_users") + private val ServedCommunitiesAction = Some("served_communities") + private val ServedPromptsAction = Some("served_prompts") private val InjectedComponent = Some("injected") private val PromotedComponent = Some("promoted") private val WhoToFollowComponent = Some("who_to_follow") private val WhoToSubscribeComponent = Some("who_to_subscribe") + private val CommunitiesToJoinComponent = Some("communities_to_join") + private val RelevancePromptComponent = Some("for_you_survey_feed") private val WithVideoDurationComponent = Some("with_video_duration") private val VideoDurationSumElement = Some("video_duration_sum") private val NumVideosElement = Some("num_videos") + def tweetTypePredicate(predicateName: String): FeatureMap => Boolean = { + val predicateIdxOpt = Home.ItemTypeIdxMap.get(predicateName) + if (predicateIdxOpt.nonEmpty) { + val predicateIdx = predicateIdxOpt.get + featureMap: FeatureMap => { + featureMap + .getOrElse(TweetTypeMetricsFeature, None) + .map { tweetTypeMetrics => + java.util.BitSet + .valueOf(tweetTypeMetrics.toArray) + .get(predicateIdx) + }.getOrElse(false) + } + } else _ => false + } def build( query: PipelineQuery, injectedTweets: Seq[ItemCandidateWithDetails], promotedTweets: Seq[ItemCandidateWithDetails], whoToFollowUsers: Seq[ItemCandidateWithDetails], - whoToSubscribeUsers: Seq[ItemCandidateWithDetails] + whoToSubscribeUsers: Seq[ItemCandidateWithDetails], + communititesToJoin: Seq[ItemCandidateWithDetails], + relevancePrompt: Seq[ItemCandidateWithDetails] ): Seq[ClientEvent] = { val baseEventNamespace = EventNamespace( section = section(query), @@ -85,25 +104,33 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder { ClientEvent( baseEventNamespace.copy(component = WhoToSubscribeComponent, action = ServedUsersAction), eventValue = count(whoToSubscribeUsers)), + ClientEvent( + baseEventNamespace + .copy(component = CommunitiesToJoinComponent, action = ServedCommunitiesAction), + eventValue = count(communititesToJoin)), + ClientEvent( + baseEventNamespace + .copy(component = RelevancePromptComponent, action = ServedPromptsAction), + eventValue = count(relevancePrompt)), ) val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map { case (tweetType, predicate) => ClientEvent( baseEventNamespace.copy(component = InjectedComponent, element = Some(tweetType)), - eventValue = count(injectedTweets, predicate, query.features.getOrElse(FeatureMap.empty)) + eventValue = count( + injectedTweets, + tweetTypePredicate(tweetType), + query.features.getOrElse(FeatureMap.empty)) ) }.toSeq - val suggestTypeServedEvents = injectedTweets - .flatMap(_.features.getOrElse(SuggestTypeFeature, None)) - .map { - InjectionScribeUtil.scribeComponent - } + val servedTypeServedEvents = injectedTweets + .map(_.features.get(ServedTypeFeature)) .groupBy(identity).map { - case (suggestType, group) => + case (servedType, group) => ClientEvent( - baseEventNamespace.copy(component = suggestType), + baseEventNamespace.copy(component = Some(servedType.originalName)), eventValue = Some(group.size.toLong)) }.toSeq @@ -119,7 +146,7 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder { ) val videoEvents = Seq(numVideosEvent, videoDurationSumEvent) - overallServedEvents ++ tweetTypeServedEvents ++ suggestTypeServedEvents ++ videoEvents + overallServedEvents ++ tweetTypeServedEvents ++ servedTypeServedEvents ++ videoEvents } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/CommonFeaturesPldrConverter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/CommonFeaturesPldrConverter.scala new file mode 100644 index 000000000..e9113905f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/CommonFeaturesPldrConverter.scala @@ -0,0 +1,140 @@ +package com.twitter.home_mixer.functional_component.side_effect + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.mysql.Client +import com.twitter.finagle.mysql.Transactions +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.util.DefaultTimer +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCommonFeatures +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCommonFeaturesAdapter +import com.twitter.home_mixer.functional_component.scorer.CommonFeaturesDataRecordFeature +import com.twitter.home_mixer.model.HomeFeatures.ClientIdFeature +import com.twitter.home_mixer.model.HomeFeatures.GuestIdFeature +import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature +import com.twitter.home_mixer.model.request.FollowingProduct +import com.twitter.home_mixer.model.request.ForYouProduct +import com.twitter.home_mixer.model.request.ScoredTweetsProduct +import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct +import com.twitter.home_mixer.model.request.SubscribedProduct +import com.twitter.home_mixer.param.HomeGlobalParams +import com.twitter.home_mixer.param.HomeMixerFlagName.DataRecordMetadataStoreConfigsYmlFlag +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.inject.annotations.Flag +import com.twitter.ml.api.DataRecordMerger +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.CandidateAndCommonFeaturesStreamingUtils +import com.twitter.timelines.ml.pldr.client.MysqlClientUtils +import com.twitter.timelines.ml.pldr.client.VersionedMetadataCacheClient +import com.twitter.timelines.ml.pldr.conversion.VersionIdAndFeatures +import com.twitter.timelines.prediction.features.large_embeddings.LargeEmbeddingsFeatures.AllCommonLargeEmbeddingsFeatures +import com.twitter.timelines.suggests.common.data_record_metadata.{thriftscala => drmd} +import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} +import com.twitter.timelines.util.stats.OptionObserver +import com.twitter.util.Try +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +@Singleton +class CommonFeaturesPldrConverter @Inject() ( + @Flag(DataRecordMetadataStoreConfigsYmlFlag) dataRecordMetadataStoreConfigsYml: String, + statsReceiver: StatsReceiver) { + + private val drMerger = new DataRecordMerger + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val metadataFetchFailedCounter = scopedStatsReceiver.counter("metadataFetchFailed") + + private val commonFeaturesPLDROptionObserver = + OptionObserver(scopedStatsReceiver.scope("commonFeaturesPLDR")) + + private lazy val dataRecordMetadataStoreClient: Option[Client with Transactions] = Try { + MysqlClientUtils.mysqlClientProvider( + MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml) + ) + }.toOption + + private lazy val versionedMetadataCacheClientOpt: Option[ + VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]] + ] = dataRecordMetadataStoreClient.map { mysqlClient => + new VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]]( + maximumSize = 1, + expireDurationOpt = None, + mysqlClient = mysqlClient, + transform = CandidateAndCommonFeaturesStreamingUtils.metadataTransformer, + statsReceiver = statsReceiver + ) + } + + versionedMetadataCacheClientOpt.foreach { + _.metadataFetchTimerTask( + CandidateAndCommonFeaturesStreamingUtils.metadataFetchKey, + metadataFetchTimer = DefaultTimer, + metadataFetchInterval = 90.seconds, + metadataFetchFailedCounter = metadataFetchFailedCounter + ) + } + + /** + * Get the common features data record converted to PLDR format with prediction request ID + * + * @param query + * @param selectedCandidates + * @return prediction request ID and the common features PLDR for the given request + */ + def getCommonFeaturesPldr( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails] + ): Option[(Long, pldr.PolyDataRecord)] = { + // Exclude unscored candidates (e.g. because of scoring QF truncation or cache reads) + val candidatesHead = selectedCandidates + .find { candidate => + candidate.features.contains(CommonFeaturesDataRecordFeature) + } + + if (candidatesHead.nonEmpty) { + val candidateFeaturesHead = candidatesHead.get.features + val predictionRequestId = candidateFeaturesHead.getOrElse(PredictionRequestIdFeature, None) + val productSurface = query.product match { + case FollowingProduct => hmt.Product.Following + case ForYouProduct => hmt.Product.ForYou + case ScoredTweetsProduct => hmt.Product.ScoredTweets + case ScoredVideoTweetsProduct => hmt.Product.ScoredVideoTweets + case SubscribedProduct => hmt.Product.Subscribed + case other => throw new UnsupportedOperationException(s"Unknown product: $other") + } + val nonMLCommonFeatures = NonMLCommonFeatures( + userId = query.getRequiredUserId, + guestId = candidateFeaturesHead.getOrElse(GuestIdFeature, None), + clientId = candidateFeaturesHead.getOrElse(ClientIdFeature, None), + countryCode = query.getCountryCode, + predictionRequestId = predictionRequestId, + productSurface = productSurface.toString, + servedTimestamp = query.queryTime.inMilliseconds + ) + val nonMLCommonFeaturesDataRecord = + NonMLCommonFeaturesAdapter.adaptToDataRecords(nonMLCommonFeatures).asScala.head + + val commonFeaturesDataRecord = candidateFeaturesHead.get(CommonFeaturesDataRecordFeature) + //Remove large embeddings from dataRecord + AllCommonLargeEmbeddingsFeatures.foreach { feature => + commonFeaturesDataRecord.tensors.remove(feature.getFeatureId) + } + drMerger.merge(commonFeaturesDataRecord, nonMLCommonFeaturesDataRecord) + + val commonFeaturesPLDROpt = CandidateAndCommonFeaturesStreamingUtils + .commonFeaturesToPolyDataRecord( + versionedMetadataCacheClientOpt = versionedMetadataCacheClientOpt, + commonFeatures = commonFeaturesDataRecord, + valueFormat = pldr.PolyDataRecord._Fields.LITE_COMPACT_DATA_RECORD, + createNew = query.params( + HomeGlobalParams.EnableCommonFeaturesDataRecordCopyDuringPldrConversionParam) + ) + + commonFeaturesPLDROptionObserver(commonFeaturesPLDROpt).flatMap { pldr => + predictionRequestId.map(_ -> pldr) + } + } else None + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala index a8feef707..f92443af1 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala @@ -21,7 +21,9 @@ case class HomeScribeClientEventSideEffect( injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier], adsCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, - whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None) + whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, + forYouCommunitiesToJoinCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, + forYouRelevancePromptCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None) extends ScribeClientEventSideEffect[PipelineQuery, Timeline] with PipelineResultSideEffect.Conditionally[ PipelineQuery, @@ -30,7 +32,7 @@ case class HomeScribeClientEventSideEffect( override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent") - override val page = "timelinemixer" + override val page = "home" override def onlyIf( query: PipelineQuery, @@ -59,8 +61,21 @@ case class HomeScribeClientEventSideEffect( val whoToSubscribeUsers = whoToSubscribeCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten + val communititesToJoin = + forYouCommunitiesToJoinCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten + + val relevancePrompt = + forYouRelevancePromptCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten + val servedEvents = ServedEventsBuilder - .build(query, injectedTweets, promotedTweets, whoToFollowUsers, whoToSubscribeUsers) + .build( + query, + injectedTweets, + promotedTweets, + whoToFollowUsers, + whoToSubscribeUsers, + communititesToJoin, + relevancePrompt) val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala index 3f19dfc99..7a528b56b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala @@ -11,15 +11,22 @@ import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature +import com.twitter.home_mixer.model.HomeFeatures.UserActionsByteArrayFeature +import com.twitter.home_mixer.model.HomeFeatures.UserActionsContainsExplicitSignalsFeature +import com.twitter.home_mixer.model.HomeFeatures.UserActionsSizeFeature +import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature +import com.twitter.home_mixer.model.PredictedScoreFeature import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct +import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixInferenceClusterParam +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher @@ -40,8 +47,10 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineMod import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.timeline_logging.thriftscala.Prediction import com.twitter.timelines.timeline_logging.{thriftscala => thrift} import com.twitter.util.Time +import java.nio.ByteBuffer import javax.inject.Inject import javax.inject.Singleton @@ -100,25 +109,9 @@ class HomeScribeServedCandidatesSideEffect @Inject() ( else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial else thrift.QueryType.Other } - val requestInfo = thrift.RequestInfo( - requestTimeMs = query.queryTime.inMilliseconds, - traceId = Trace.id.traceId.toLong, - userId = query.getOptionalUserId, - clientAppId = query.clientContext.appId, - hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)), - parentId = Some(Trace.id.parentId.toLong), - spanId = Some(Trace.id.spanId.toLong), - timelineType = Some(timelineType), - ipAddress = query.clientContext.ipAddress, - userAgent = query.clientContext.userAgent, - queryType = queryType, - requestProvenance = requestProvenance, - languageCode = query.clientContext.languageCode, - countryCode = query.clientContext.countryCode, - requestEndTimeMs = Some(Time.now.inMilliseconds), - servedRequestId = query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)), - requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)) - ) + + val phoenixCluster = + s"phoenix.${query.params(PhoenixInferenceClusterParam).toString.toLowerCase}." val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = selectedCandidates.flatMap { @@ -140,11 +133,46 @@ class HomeScribeServedCandidatesSideEffect @Inject() ( case _ => Seq.empty }.toMap - response.instructions.zipWithIndex + val userActions = selectedCandidates.collectFirst { + case candidate if candidate.features.getOrElse(UserActionsSizeFeature, None).isDefined => + ( + candidate.features.getOrElse(UserActionsSizeFeature, None), + candidate.features.getOrElse(UserActionsContainsExplicitSignalsFeature, false) + ) + } + + val userActionsByteArrayOpt = + query.features.flatMap(_.getOrElse(UserActionsByteArrayFeature, None)) + val userActionsBuffer = userActionsByteArrayOpt.map(ua => ByteBuffer.wrap(ua)) + + val requestInfo = thrift.RequestInfo( + requestTimeMs = query.queryTime.inMilliseconds, + traceId = Trace.id.traceId.toLong, + userId = query.getOptionalUserId, + clientAppId = query.clientContext.appId, + hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)), + parentId = Some(Trace.id.parentId.toLong), + spanId = Some(Trace.id.spanId.toLong), + timelineType = Some(timelineType), + ipAddress = query.clientContext.ipAddress, + userAgent = query.clientContext.userAgent, + queryType = queryType, + requestProvenance = requestProvenance, + languageCode = query.clientContext.languageCode, + countryCode = query.clientContext.countryCode, + requestEndTimeMs = Some(Time.now.inMilliseconds), + servedRequestId = query.features.flatMap(_.getOrElse(ServedIdFeature, None)), + requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)), + userActionsSize = userActions.flatMap(_._1), + userActionsContainsExplicitSignals = userActions.map(_._2), + userActions = userActionsBuffer + ) + + response.instructions .collect { - case (AddEntriesTimelineInstruction(entries), index) => - entries.collect { - case entry: TweetItem if entry.promotedMetadata.isDefined => + case AddEntriesTimelineInstruction(entries) => + entries.zipWithIndex.collect { + case (entry: TweetItem, index) if entry.promotedMetadata.isDefined => val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index) Seq( thrift.EntryInfo( @@ -156,9 +184,8 @@ class HomeScribeServedCandidatesSideEffect @Inject() ( verticalSize = Some(1), displayType = Some(entry.displayType.toString), details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails)) - ) - ) - case entry: TweetItem => + )) + case (entry: TweetItem, index) => val candidate = tweetIdToItemCandidateMap(entry.id) val tweetDetails = TweetDetailsMarshaller(entry, candidate) Seq( @@ -171,13 +198,22 @@ class HomeScribeServedCandidatesSideEffect @Inject() ( verticalSize = Some(1), score = candidate.features.getOrElse(ScoreFeature, None), displayType = Some(entry.displayType.toString), - details = Some(thrift.ItemDetails.TweetDetails(tweetDetails)) - ) - ) - case module: TimelineModule + details = Some(thrift.ItemDetails.TweetDetails(tweetDetails)), + predictionScores = Some(extractPredictionScores(candidate, phoenixCluster)), + sourceSignal = candidate.features.getOrElse(SourceSignalFeature, None).map { + signal => + thrift.SourceSignal( + id = Some(signal.id), + signalType = signal.signalType, + signalEntity = signal.signalEntity, + authorId = signal.authorId, + ) + } + )) + case (module: TimelineModule, _) if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString => - module.items.collect { - case ModuleItem(entry: UserItem, _, _) => + module.items.zipWithIndex.collect { + case (ModuleItem(entry: UserItem, _, _, _), index) => val candidate = userIdToItemCandidateMap(entry.id) val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate) thrift.EntryInfo( @@ -191,10 +227,10 @@ class HomeScribeServedCandidatesSideEffect @Inject() ( details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails)) ) } - case module: TimelineModule + case (module: TimelineModule, _) if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString => - module.items.collect { - case ModuleItem(entry: UserItem, _, _) => + module.items.zipWithIndex.collect { + case (ModuleItem(entry: UserItem, _, _, _), index) => val candidate = userIdToItemCandidateMap(entry.id) val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate) thrift.EntryInfo( @@ -208,11 +244,11 @@ class HomeScribeServedCandidatesSideEffect @Inject() ( details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails)) ) } - case module: TimelineModule + case (module: TimelineModule, _) if module.sortIndex.isDefined && module.items.headOption.exists( _.item.isInstanceOf[TweetItem]) => - module.items.collect { - case ModuleItem(entry: TweetItem, _, _) => + module.items.zipWithIndex.collect { + case (ModuleItem(entry: TweetItem, _, _, _), index) => val candidate = tweetIdToItemCandidateMap(entry.id) thrift.EntryInfo( id = entry.id, @@ -221,7 +257,15 @@ class HomeScribeServedCandidatesSideEffect @Inject() ( entryType = thrift.EntryType.ConversationModule, sortIndex = module.sortIndex, score = candidate.features.getOrElse(ScoreFeature, None), - displayType = Some(entry.displayType.toString) + displayType = Some(entry.displayType.toString), + predictionScores = Some(extractPredictionScores(candidate, phoenixCluster)), + sourceSignal = + candidate.features.getOrElse(SourceSignalFeature, None).map { signal => + thrift.SourceSignal( + id = Some(signal.id), + signalType = signal.signalType + ) + } ) } case _ => Seq.empty @@ -242,4 +286,23 @@ class HomeScribeServedCandidatesSideEffect @Inject() ( override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert() ) + + private def extractPredictionScores( + candidate: ItemCandidateWithDetails, + phoenixCluster: String + ): Set[Prediction] = { + val naviPredictionScores = PredictedScoreFeature.PredictedScoreFeatureSet.map { feature => + Prediction(Some(feature.featureName), candidate.features.getOrElse(feature, None)) + } + + val phoenixPredictionScores = + PhoenixPredictedScoreFeature.PhoenixPredictedScoreFeatureSet.map { feature => + Prediction( + Some(phoenixCluster + feature.featureName), + candidate.features.getOrElse(feature, None) + ) + } + + (naviPredictionScores ++ phoenixPredictionScores).filter(_.score.isDefined) + } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala index c0437767e..7d934554f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala @@ -1,6 +1,9 @@ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.eventbus.client.EventBusPublisher +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature +import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.SubscribedProduct @@ -32,7 +35,8 @@ object PublishClientSentImpressionsEventBusSideEffect { */ @Singleton class PublishClientSentImpressionsEventBusSideEffect @Inject() ( - eventBusPublisher: EventBusPublisher[PublishedImpressionList]) + eventBusPublisher: EventBusPublisher[PublishedImpressionList], + statsReceiver: StatsReceiver) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] with PipelineResultSideEffect.Conditionally[ PipelineQuery with HasSeenTweetIds, @@ -43,6 +47,8 @@ class PublishClientSentImpressionsEventBusSideEffect @Inject() ( override val identifier: SideEffectIdentifier = SideEffectIdentifier("PublishClientSentImpressionsEventBus") + private val seenIdsStatsReceiver = statsReceiver.scope(identifier.toString).scope("SeenIds") + override def onlyIf( query: PipelineQuery with HasSeenTweetIds, selectedCandidates: Seq[CandidateWithDetails], @@ -61,7 +67,17 @@ class PublishClientSentImpressionsEventBusSideEffect @Inject() ( case SubscribedProduct => HomeSubscribedSurfaceArea case _ => None } + + val device = query.clientContext.appId.getOrElse(0L).toString + query.seenTweetIds.map { seenTweetIds => + val getNewer = query.features.map(_.getOrElse(GetNewerFeature, false)).getOrElse(false) + val getOlder = query.features.map(_.getOrElse(GetOlderFeature, false)).getOrElse(false) + val requestType = if (getNewer) "newer" else if (getOlder) "older" else "none" + seenIdsStatsReceiver + .scope(query.product.identifier.name).scope(requestType).stat(device) + .add(seenTweetIds.distinct.size) + seenTweetIds.map { tweetId => Impression( tweetId = tweetId, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala index 5a1bb0f6b..efe7f19c8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala @@ -1,10 +1,33 @@ package com.twitter.home_mixer.functional_component.side_effect +import com.twitter.home_mixer.functional_component.decorator.EntryPointPivotModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.ForYouTweetCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.KeywordTrendsModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.StoriesModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.TuneFeedModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.VideoCarouselModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.urt.builder.BookmarksModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.urt.builder.PinnedTweetsModuleCandidateDecorator import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.model.PredictedDwellScoreFeature +import com.twitter.home_mixer.model.PredictedFavoriteScoreFeature +import com.twitter.home_mixer.model.PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature +import com.twitter.home_mixer.model.PredictedGoodClickConvoDescUamGt2ScoreFeature +import com.twitter.home_mixer.model.PredictedGoodProfileClickScoreFeature +import com.twitter.home_mixer.model.PredictedNegativeFeedbackV2ScoreFeature +import com.twitter.home_mixer.model.PredictedReplyEngagedByAuthorScoreFeature +import com.twitter.home_mixer.model.PredictedReplyScoreFeature +import com.twitter.home_mixer.model.PredictedRetweetScoreFeature +import com.twitter.home_mixer.model.PredictedShareScoreFeature +import com.twitter.home_mixer.model.PredictedVideoQualityViewScoreFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnablePersistenceDebug import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.pipeline.candidate.communities_to_join.CommunitiesToJoinCandidateDecorator +import com.twitter.product_mixer.component_library.pipeline.candidate.job.RecommendedJobsCandidateDecorator +import com.twitter.product_mixer.component_library.pipeline.candidate.recruiting_organization.RecommendedRecruitingOrganizationsCandidateDecorator import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -17,18 +40,20 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.ReplaceEntr import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowCoverInstruction import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.PromptItem import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem +import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.PromotedMetadata import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.clients.persistence.EntryWithItemIds import com.twitter.timelinemixer.clients.persistence.ItemIds import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.timelines.persistence.thriftscala.TweetScoreV1 import com.twitter.timelines.persistence.{thriftscala => persistence} +import com.twitter.timelineservice.model.PredictedScores import com.twitter.timelineservice.model.TimelineQuery import com.twitter.timelineservice.model.TimelineQueryOptions -import com.twitter.timelineservice.model.TweetScore +import com.twitter.timelineservice.model.TweetScoreV1 import com.twitter.timelineservice.model.core.TimelineKind import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.util.Time @@ -36,25 +61,6 @@ import com.twitter.{timelineservice => tls} import javax.inject.Inject import javax.inject.Singleton -object UpdateTimelinesPersistenceStoreSideEffect { - val EmptyItemIds = ItemIds( - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None) -} - /** * Side effect that updates the Timelines Persistence Store (Manhattan) with the entries being returned. */ @@ -75,6 +81,9 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( case ForYouProduct => TimelineKind.home case other => throw new UnsupportedOperationException(s"Unknown product: $other") } + + val debugEnabled = inputs.query.params(EnablePersistenceDebug) + val timelineQuery = TimelineQuery( id = inputs.query.getRequiredUserId, kind = timelineKind, @@ -100,108 +109,126 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( case AddEntriesTimelineInstruction(entries) => entries.collect { // includes tweets, tweet previews, and promoted tweets - case entry: TweetItem if entry.sortIndex.isDefined => { - Seq( - buildTweetEntryWithItemIds( - tweetIdToItemCandidateMap(entry.id), - entry.sortIndex.get - )) - } - // tweet conversation modules are flattened to individual tweets in the persistence store - case module: TimelineModule - if module.sortIndex.isDefined && module.items.headOption.exists( - _.item.isInstanceOf[TweetItem]) => - module.items.map { item => - buildTweetEntryWithItemIds( - tweetIdToItemCandidateMap(item.item.id.asInstanceOf[Long]), - module.sortIndex.get) - } - case module: TimelineModule - if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString => - val userIds = module.items - .map(item => - UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId = - Some(item.item.id.asInstanceOf[Long]))) + case entry: TweetItem if entry.sortIndex.isDefined => + val tweetEntry = buildTweetEntryWithItemIds( + tweetIdToItemCandidateMap(entry.id), + entry.sortIndex.get, + debugEnabled, + entry.promotedMetadata + ) + Seq(tweetEntry) + case entry: PromptItem => Seq( EntryWithItemIds( - entityIdType = EntityIdType.WhoToFollow, - sortIndex = module.sortIndex.get, - size = module.items.size.toShort, - itemIds = Some(userIds) + // Temporarily use annotation here as entity type until we come up with a proper one + entityIdType = EntityIdType.Annotation, + sortIndex = entry.sortIndex.getOrElse(0L), + size = 1, + itemIds = None )) - case module: TimelineModule - if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString => - val userIds = module.items - .map(item => - UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId = - Some(item.item.id.asInstanceOf[Long]))) - Seq( - EntryWithItemIds( - entityIdType = EntityIdType.WhoToSubscribe, + case module: TimelineModule if module.sortIndex.isDefined && module.items.nonEmpty => + if (module.entryNamespace == ForYouTweetCandidateDecorator.ConvoModuleEntryNamespace) { + module.items.map { item => + buildTweetEntryWithItemIds( + tweetIdToItemCandidateMap(item.item.id.asInstanceOf[Long]), + module.sortIndex.get, + debugEnabled + ) + } + } else if (module.entryNamespace == StoriesModuleCandidateDecorator.TrendsEntryNamespace + || module.entryNamespace == KeywordTrendsModuleCandidateDecorator.KeywordTrendsEntryNamespace) { + Seq( + EntryWithItemIds( + entityIdType = EntityIdType.Trends, + sortIndex = module.sortIndex.get, + size = module.items.size.toShort, + itemIds = None + ) + ) + } else { + val moduleItems = module.items.map(_.item.id.asInstanceOf[Long]) + + val (entityId, items) = module.entryNamespace.toString match { + case WhoToFollowCandidateDecorator.EntryNamespaceString => + val itemIds = moduleItems.map(id => ItemIds(userId = Some(id))) + (EntityIdType.WhoToFollow, itemIds) + case WhoToSubscribeCandidateDecorator.EntryNamespaceString => + val itemIds = moduleItems.map(id => ItemIds(userId = Some(id))) + (EntityIdType.WhoToSubscribe, itemIds) + case CommunitiesToJoinCandidateDecorator.EntryNamespaceString => + val itemIds = moduleItems.map(id => ItemIds(communityId = Some(id))) + (EntityIdType.CommunityModule, itemIds) + case RecommendedJobsCandidateDecorator.EntryNamespaceString => + val itemIds = moduleItems.map(id => ItemIds(jobId = Some(id))) + (EntityIdType.JobModule, itemIds) + case RecommendedRecruitingOrganizationsCandidateDecorator.EntryNamespaceString => + val itemIds = + moduleItems.map(id => ItemIds(recruitingOrganizationId = Some(id))) + (EntityIdType.RecruitingOrganizationModule, itemIds) + case BookmarksModuleCandidateDecorator.entryNamespaceString => + (EntityIdType.BookmarksModule, Seq.empty) + case PinnedTweetsModuleCandidateDecorator.entryNamespaceString => + (EntityIdType.PinnedTweetsModule, Seq.empty) + case VideoCarouselModuleCandidateDecorator.entryNamespaceString => + (EntityIdType.VideoCarouselModule, Seq.empty) + case EntryPointPivotModuleCandidateDecorator.EntryNamespaceString => + (EntityIdType.EntryPointPivot, Seq.empty) + case TuneFeedModuleCandidateDecorator.EntryNamespaceString => + (EntityIdType.TuneFeedModule, Seq.empty) + case other => throw new IllegalStateException("Invalid namespace: " + other) + } + + val entry = EntryWithItemIds( + entityIdType = entityId, sortIndex = module.sortIndex.get, size = module.items.size.toShort, - itemIds = Some(userIds) - )) + itemIds = Some(items) + ) + Seq(entry) + } }.flatten case ShowCoverInstruction(cover) => - Seq( - EntryWithItemIds( - entityIdType = EntityIdType.Prompt, - sortIndex = cover.sortIndex.get, - size = 1, - itemIds = None - ) + val entry = EntryWithItemIds( + entityIdType = EntityIdType.Prompt, + sortIndex = cover.sortIndex.get, + size = 1, + itemIds = None ) - case ReplaceEntryTimelineInstruction(entry) => + Seq(entry) + case ReplaceEntryTimelineInstruction(replace) => val namespaceLength = TweetItem.TweetEntryNamespace.toString.length - Seq( - EntryWithItemIds( - entityIdType = EntityIdType.Tweet, - sortIndex = entry.sortIndex.get, - size = 1, - itemIds = Some( - Seq( - ItemIds( - tweetId = - entry.entryIdToReplace.map(e => e.substring(namespaceLength + 1).toLong), - sourceTweetId = None, - quoteTweetId = None, - sourceAuthorId = None, - quoteAuthorId = None, - inReplyToTweetId = None, - inReplyToAuthorId = None, - semanticCoreId = None, - articleId = None, - hasRelevancePrompt = None, - promptData = None, - tweetScore = None, - entryIdToReplace = entry.entryIdToReplace, - tweetReactiveData = None, - userId = None - ) - )) - ) + val itemId = ItemIds( + tweetId = replace.entryIdToReplace.map(_.substring(namespaceLength + 1).toLong), + entryIdToReplace = replace.entryIdToReplace, ) - + val entry = EntryWithItemIds( + entityIdType = EntityIdType.Tweet, + sortIndex = replace.sortIndex.get, + size = 1, + itemIds = Some(Seq(itemId)) + ) + Seq(entry) }.flatten + val servedId = inputs.query.features.flatMap(_.getOrElse(ServedIdFeature, None)) + val response = TimelineResponseV3( clientPlatform = timelineQuery.clientPlatform, servedTime = Time.now, requestType = requestTypeFromQuery(inputs.query), - entries = entries) + entries = entries, + servedId = servedId, + ) Stitch.callFuture(timelineResponseBatchesClient.insertResponse(timelineQuery, response)) } else Stitch.Unit } - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) - private def buildTweetEntryWithItemIds( candidate: ItemCandidateWithDetails, - sortIndex: Long + sortIndex: Long, + debug: Boolean, + promotedMetadata: Option[PromotedMetadata] = None ): EntryWithItemIds = { val features = candidate.features val sourceAuthorId = @@ -211,11 +238,39 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( if (features.getOrElse(QuotedTweetIdFeature, None).nonEmpty) features.getOrElse(SourceUserIdFeature, None) else None - val tweetScore = features.getOrElse(ScoreFeature, None).map { score => - TweetScore.fromThrift(persistence.TweetScore.TweetScoreV1(TweetScoreV1(score))) - } + val tweetScore = features.getOrElse(ScoreFeature, None) + val debugInfo = features.getOrElse(DebugStringFeature, None) + val predictionRequestId = features.getOrElse(PredictionRequestIdFeature, None) + val servedType = candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined).name + val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false) + val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet + val grokAnnotations = if (debug) features.getOrElse(GrokAnnotationsFeature, None) else None + val topics = grokAnnotations.map(_.topics) + val tags = grokAnnotations.map(_.tags) + val impressionId = if (debug) promotedMetadata.flatMap(_.impressionString) else None + val predictedScores = + if (debug) + Some( + PredictedScores( + favoriteScore = features.getOrElse(PredictedFavoriteScoreFeature, None), + replyScore = features.getOrElse(PredictedReplyScoreFeature, None), + retweetScore = features.getOrElse(PredictedRetweetScoreFeature, None), + replyEngagedByAuthorScore = + features.getOrElse(PredictedReplyEngagedByAuthorScoreFeature, None), + goodClickConvoDescFavoritedOrRepliedScore = features + .getOrElse(PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, None), + goodClickConvoDescUamGt2Score = + features.getOrElse(PredictedGoodClickConvoDescUamGt2ScoreFeature, None), + goodProfileClickScore = features.getOrElse(PredictedGoodProfileClickScoreFeature, None), + videoQualityViewScore = features.getOrElse(PredictedVideoQualityViewScoreFeature, None), + shareScore = features.getOrElse(PredictedShareScoreFeature, None), + dwellScore = features.getOrElse(PredictedDwellScoreFeature, None), + negativeFeedbackV2Score = + features.getOrElse(PredictedNegativeFeedbackV2ScoreFeature, None) + )) + else None - val itemIds = ItemIds( + val itemId = ItemIds( tweetId = Some(candidate.candidateIdLong), sourceTweetId = features.getOrElse(SourceTweetIdFeature, None), quoteTweetId = features.getOrElse(QuotedTweetIdFeature, None), @@ -224,23 +279,24 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( inReplyToTweetId = features.getOrElse(InReplyToTweetIdFeature, None), inReplyToAuthorId = features.getOrElse(DirectedAtUserIdFeature, None), semanticCoreId = features.getOrElse(SemanticCoreIdFeature, None), - articleId = None, - hasRelevancePrompt = None, - promptData = None, - tweetScore = tweetScore, - entryIdToReplace = None, - tweetReactiveData = None, - userId = None + tweetScore = tweetScore.map( + TweetScoreV1( + _, + Some(servedType), + debugInfo, + predictionRequestId, + topics, + tags, + predictedScores) + ), + impressionId = impressionId ) - val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false) - val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet - EntryWithItemIds( entityIdType = entityType, sortIndex = sortIndex, size = 1.toShort, - itemIds = Some(Seq(itemIds)) + itemIds = Some(Seq(itemId)) ) } @@ -260,4 +316,6 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( case (feature, requestType) if features.getOrElse(feature, false) => requestType }.getOrElse(persistence.RequestType.Other) } + + override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8)) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.scala index 81a9abf2b..d998d7c77 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.scala @@ -19,7 +19,8 @@ class HomeMixerDebugParamsUnmarshaller @Inject() ( }, debugOptions = debugParams.debugOptions.map { options => HomeMixerDebugOptions( - requestTimeOverride = options.requestTimeOverrideMillis.map(Time.fromMilliseconds) + requestTimeOverride = options.requestTimeOverrideMillis.map(Time.fromMilliseconds), + showIntermediateLogs = options.showIntermediateLogs.orElse(Some(false)) ) } ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala index ec3a183b9..d75baeee6 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala @@ -2,9 +2,9 @@ package com.twitter.home_mixer.marshaller.request import com.twitter.home_mixer.model.request.FollowingProductContext import com.twitter.home_mixer.model.request.ForYouProductContext -import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext -import com.twitter.home_mixer.model.request.ListTweetsProductContext +import com.twitter.home_mixer.model.request.HeavyRankerScoresProductContext import com.twitter.home_mixer.model.request.ScoredTweetsProductContext +import com.twitter.home_mixer.model.request.ScoredVideoTweetsProductContext import com.twitter.home_mixer.model.request.SubscribedProductContext import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.model.marshalling.request.ProductContext @@ -26,8 +26,7 @@ class HomeMixerProductContextUnmarshaller @Inject() ( ForYouProductContext( deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), seenTweetIds = p.seenTweetIds, - dspClientContext = p.dspClientContext, - pushToHomeTweetId = p.pushToHomeTweetId + dspClientContext = p.dspClientContext ) case t.ProductContext.ListManagement(p) => throw new UnsupportedOperationException(s"This product is no longer used") @@ -36,26 +35,36 @@ class HomeMixerProductContextUnmarshaller @Inject() ( deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), seenTweetIds = p.seenTweetIds, servedTweetIds = p.servedTweetIds, - backfillTweetIds = p.backfillTweetIds + backfillTweetIds = p.backfillTweetIds, + signupCountryCode = p.signupCountryCode, + allowForYouRecommendations = p.allowForYouRecommendations, + signupSource = None, // not exposed in thrift interface + followerCount = p.followerCount ) - case t.ProductContext.ListTweets(p) => - ListTweetsProductContext( - listId = p.listId, + case t.ProductContext.ScoredVideoTweets(p) => + ScoredVideoTweetsProductContext( deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), - dspClientContext = p.dspClientContext + seenTweetIds = p.seenTweetIds, + videoType = p.videoType, + pinnedRelatedTweetIds = p.pinnedRelatedTweetIds, + scorePinnedTweetsOnly = p.scorePinnedTweetsOnly, + immersiveClientMetadata = p.immersiveClientMetadata ) + case t.ProductContext.ListTweets(p) => + throw new UnsupportedOperationException(s"This product is no longer used") case t.ProductContext.ListRecommendedUsers(p) => - ListRecommendedUsersProductContext( - listId = p.listId, - selectedUserIds = p.selectedUserIds, - excludedUserIds = p.excludedUserIds, - listName = p.listName - ) + throw new UnsupportedOperationException(s"This product is no longer used") case t.ProductContext.Subscribed(p) => SubscribedProductContext( deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), seenTweetIds = p.seenTweetIds, ) + case t.ProductContext.HeavyRankerScores(p) => + HeavyRankerScoresProductContext( + deviceContext = p.tweetScoringRequestContext + .flatMap(_.deviceContext.map(deviceContextUnmarshaller(_))), + tweetIds = p.tweetIds + ) case t.ProductContext.UnknownUnionField(field) => throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}") } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala index f5d0d002b..fdc079f52 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala @@ -2,9 +2,9 @@ package com.twitter.home_mixer.marshaller.request import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct +import com.twitter.home_mixer.model.request.HeavyRankerScoresProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct +import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.model.marshalling.request.Product @@ -20,9 +20,13 @@ class HomeMixerProductUnmarshaller @Inject() () { case t.Product.ListManagement => throw new UnsupportedOperationException(s"This product is no longer used") case t.Product.ScoredTweets => ScoredTweetsProduct - case t.Product.ListTweets => ListTweetsProduct - case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct + case t.Product.ScoredVideoTweets => ScoredVideoTweetsProduct + case t.Product.ListTweets => + throw new UnsupportedOperationException(s"This product is no longer used") + case t.Product.ListRecommendedUsers => + throw new UnsupportedOperationException(s"This product is no longer used") case t.Product.Subscribed => SubscribedProduct + case t.Product.HeavyRankerScores => HeavyRankerScoresProduct case t.Product.EnumUnknownProduct(value) => throw new UnsupportedOperationException(s"Unknown product: $value") } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.bazel index efcad840b..062295b56 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.bazel @@ -5,8 +5,8 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala index 8e1c475d5..7add7be68 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala @@ -3,7 +3,7 @@ package com.twitter.home_mixer.marshaller.timeline_logging import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.GeneralContextTypeMarshaller @@ -39,7 +39,7 @@ object TweetDetailsMarshaller { thriftlog.TweetDetails( sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), socialContextType = socialContextType, - suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name), + suggestType = Some(candidate.features.get(ServedTypeFeature).name), authorId = candidate.features.getOrElse(AuthorIdFeature, None), sourceAuthorId = candidate.features.getOrElse(SourceUserIdFeature, None) ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel index 65ece62a3..20f0ac5c7 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel @@ -4,38 +4,42 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", + "3rdparty/jvm/io/grpc:grpc-protobuf", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + "search/search-router/thrift/src/main/thrift:thrift-scala", "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/ml/api/constant", - "src/scala/com/twitter/ml/api:api-base", "src/scala/com/twitter/timelines/prediction/features/common", + "src/scala/com/twitter/timelines/prediction/features/conversation_features", + "src/scala/com/twitter/timelines/prediction/features/engaged_language_features", "src/scala/com/twitter/timelines/prediction/features/recap", "src/scala/com/twitter/timelines/prediction/features/request_context", - "src/thrift/com/twitter/escherbird:tweet-annotation-scala", + "src/scala/com/twitter/timelines/prediction/features/user_health", "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala", - "src/thrift/com/twitter/timelines/impression:thrift-scala", "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", + "strato/config/src/thrift/com/twitter/strato/columns/content_understanding:content_understanding-scala", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", "topic-social-proof/server/src/main/thrift:thrift-scala", "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", + "user_history_transformer/service/src/main/java/com/x/user_action_sequence", ], exports = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/timelines/impression:thrift-scala", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.scala index 85154e55b..b279d5a8b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.scala @@ -12,14 +12,18 @@ import com.twitter.timelines.configapi.FSParam /** * Include a clear cache timeline instruction when we satisfy these criteria: - * - Request Provenance is "pull to refresh" - * - Atleast N non-ad tweet entries in the response + * - Request Provenance + * - At least N non-ad tweet entries in the response * * This is to ensure that we have sufficient new content to justify jumping users to the * top of the new timelines response and don't add unnecessary load to backend systems */ case class ClearCacheIncludeInstruction( - enableParam: FSParam[Boolean], + ptrEnableParam: FSParam[Boolean], + coldStartEnableParam: FSParam[Boolean], + warmStartEnableParam: FSParam[Boolean], + manualRefreshEnableParam: FSParam[Boolean], + navigateEnableParam: FSParam[Boolean], minEntriesParam: FSBoundedParam[Int]) extends IncludeInstruction[PipelineQuery with HasDeviceContext] { @@ -27,10 +31,19 @@ case class ClearCacheIncludeInstruction( query: PipelineQuery with HasDeviceContext, entries: Seq[TimelineEntry] ): Boolean = { - val enabled = query.params(enableParam) + val requestContext = query.deviceContext.flatMap(_.requestContextValue) - val ptr = - query.deviceContext.flatMap(_.requestContextValue).contains(RequestContext.PullToRefresh) + val ptrEnabled = + query.params(ptrEnableParam) && requestContext.contains(RequestContext.PullToRefresh) + val coldStartEnabled = + query.params(coldStartEnableParam) && requestContext.contains(RequestContext.Launch) + val warmStartEnabled = + query.params(warmStartEnableParam) && requestContext.contains(RequestContext.Foreground) + val manualRefreshEnabled = + query.params(manualRefreshEnableParam) && requestContext.contains( + RequestContext.ManualRefresh) + val navigateEnabled = + query.params(navigateEnableParam) && requestContext.contains(RequestContext.Navigate) val minTweets = query.params(minEntriesParam) <= entries.collect { case item: TweetItem if item.promotedMetadata.isEmpty => 1 @@ -38,6 +51,6 @@ case class ClearCacheIncludeInstruction( module.items.size }.sum - enabled && ptr && minTweets + (ptrEnabled || coldStartEnabled || warmStartEnabled || manualRefreshEnabled || navigateEnabled) && minTweets } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala index f141d67d1..6f07e760b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala @@ -47,6 +47,8 @@ object ContentFeatures { None, None, None, + None, + None, None ) @@ -94,7 +96,9 @@ object ContentFeatures { conversationControl = ebFeatures.conversationControl, // media and selfThreadMetadata not carried by ThriftTweetFeatures media = None, - selfThreadMetadata = None + selfThreadMetadata = None, + hasImage = None, + hasVideo = None ) } @@ -141,4 +145,6 @@ case class ContentFeatures( selfThreadMetadata: Option[tp.SelfThreadMetadata], tokens: Option[Seq[String]], conversationControl: Option[tp.ConversationControl], + hasImage: Option[Boolean], + hasVideo: Option[Boolean], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GrokTopics.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GrokTopics.scala new file mode 100644 index 000000000..50061fe24 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GrokTopics.scala @@ -0,0 +1,38 @@ +package com.twitter.home_mixer.model + +object GrokTopics { + val GrokCategoryIdToNameMap = Map( + -> "Sports", + -> "Anime", + -> "Celebrity", + -> "Music", + -> "News", + -> "Business & Finance", + -> "Cryptocurrency", + -> "Technology", + -> "Science", + -> "Gaming", + -> "Movies & TV", + -> "Travel", + -> "Food", + -> "Health & Fitness", + -> "Memes", + -> "Art", + -> "Fashion", + -> "Religion", + -> "Shopping", + -> "Cars", + -> "Aviation", + -> "Motorcycles", + -> "Beauty", + -> "Nature & Outdoors", + -> "Pets", + -> "Relationships", + -> "Home & Garden", + -> "Career", + -> "Dance", + -> "Education", + -> "Podcasts", + -> "Streaming" + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala index fe085b1f9..a17c9fdb5 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala @@ -1,9 +1,14 @@ package com.twitter.home_mixer.model import com.twitter.core_workflows.user_model.{thriftscala => um} +import com.twitter.dal.personal_data.thriftjava.PersonalDataType import com.twitter.dal.personal_data.{thriftjava => pd} import com.twitter.gizmoduck.{thriftscala => gt} +import com.twitter.home_mixer.model.candidate_source.SourceSignal +import com.twitter.home_mixer.model.signup.SignupSource import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.timelines.render.{thriftscala => urt} +import com.twitter.mediaservices.commons.thriftscala.MediaCategory import com.twitter.ml.api.constant.SharedFeatures import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature @@ -13,34 +18,46 @@ import com.twitter.product_mixer.core.feature.datarecord.DataRecordFeature import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible import com.twitter.product_mixer.core.feature.datarecord.LongDiscreteDataRecordCompatible +import com.twitter.product_mixer.core.feature.datarecord.SparseBinaryDataRecordCompatible +import com.twitter.product_mixer.core.feature.datarecord.SparseContinuousDataRecordCompatible +import com.twitter.product_mixer.core.feature.datarecord.StringDataRecordCompatible import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleDisplayType import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.search.common.features.{thriftscala => sc} import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.strato.columns.content_understanding.thriftscala.AnnotatedVideoDeserialized import com.twitter.timelinemixer.clients.manhattan.DismissInfo import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 import com.twitter.timelinemixer.injection.model.candidate.AudioSpaceMetaData import com.twitter.timelines.conversation_features.v1.thriftscala.ConversationFeatures -import com.twitter.timelines.impression.{thriftscala => imp} import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} import com.twitter.timelines.model.UserId import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import com.twitter.timelines.prediction.features.conversation_features +import com.twitter.timelines.prediction.features.engaged_language_features.LanguageFeatures import com.twitter.timelines.prediction.features.engagement_features.EngagementDataRecordFeatures import com.twitter.timelines.prediction.features.recap.RecapFeatures import com.twitter.timelines.prediction.features.request_context.RequestContextFeatures -import com.twitter.timelines.service.{thriftscala => tst} +import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures import com.twitter.timelineservice.model.FeedbackEntry -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} import com.twitter.timelineservice.suggests.{thriftscala => st} import com.twitter.tsp.{thriftscala => tsp} import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} +import com.twitter.util.Duration import com.twitter.util.Time +import com.x.user_action_sequence.UserActionSequence object HomeFeatures { // Candidate Features object AncestorsFeature extends Feature[TweetCandidate, Seq[ta.TweetAncestor]] object AudioSpaceMetaDataFeature extends Feature[TweetCandidate, Option[AudioSpaceMetaData]] - object TwitterListIdFeature extends Feature[TweetCandidate, Option[Long]] + object ListIdFeature extends Feature[TweetCandidate, Option[Long]] + object ListNameFeature extends Feature[TweetCandidate, Option[String]] + object ValidLikedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] + object BookmarkedTweetTimestamp extends Feature[TweetCandidate, Option[Long]] + object ArticleIdFeature extends Feature[TweetCandidate, Option[Long]] + object ArticlePreviewTextFeature extends Feature[TweetCandidate, Option[String]] /** * For Retweets, this should refer to the retweeting user. Use [[SourceUserIdFeature]] if you want to know @@ -53,18 +70,21 @@ object HomeFeatures { override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.UserId) } + object AuthorAccountAge extends Feature[TweetCandidate, Option[Duration]] object AuthorIsBlueVerifiedFeature extends Feature[TweetCandidate, Boolean] object AuthorIsGoldVerifiedFeature extends Feature[TweetCandidate, Boolean] object AuthorIsGrayVerifiedFeature extends Feature[TweetCandidate, Boolean] object AuthorIsLegacyVerifiedFeature extends Feature[TweetCandidate, Boolean] object AuthorIsCreatorFeature extends Feature[TweetCandidate, Boolean] object AuthorIsProtectedFeature extends Feature[TweetCandidate, Boolean] - + object AuthorFollowersFeature extends Feature[TweetCandidate, Option[Long]] + object ViralContentCreatorFeature extends Feature[TweetCandidate, Boolean] + object GrokContentCreatorFeature extends Feature[TweetCandidate, Boolean] + object GorkContentCreatorFeature extends Feature[TweetCandidate, Boolean] object AuthoredByContextualUserFeature extends Feature[TweetCandidate, Boolean] object CachedCandidatePipelineIdentifierFeature extends Feature[TweetCandidate, Option[String]] - object CandidateSourceIdFeature - extends Feature[TweetCandidate, Option[cts.CandidateTweetSourceId]] object ConversationFeature extends Feature[TweetCandidate, Option[ConversationFeatures]] + object AuthorSafetyLabels extends Feature[TweetCandidate, Option[Seq[String]]] /** * This field should be set to the focal Tweet's tweetId for all tweets which are expected to @@ -96,6 +116,116 @@ object HomeFeatures { } object FavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object FeedbackHistoryFeature extends Feature[TweetCandidate, Seq[FeedbackEntry]] + // A boolean feature indicating whether the latest feedback timestamp till now is newer than the cached scored tweets TTL + object HasRecentFeedbackSinceCacheTtlFeature extends Feature[PipelineQuery, Boolean] + + object SlopAuthorFeature extends Feature[TweetCandidate, Boolean] + + object SlopAuthorScoreFeature + extends DataRecordOptionalFeature[TweetCandidate, Double] + with DoubleDataRecordCompatible { + override val featureName: String = RecapFeatures.SLOP_AUTHOR_SCORE.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty + } + + object GrokTranslatedPostIsCachedFeature extends Feature[TweetCandidate, Boolean] + + object GrokVideoMetadataFeature + extends FeatureWithDefaultOnFailure[TweetCandidate, Option[AnnotatedVideoDeserialized]] { + override def defaultValue: Option[AnnotatedVideoDeserialized] = None + } + + object GrokCategoryDataRecordFeature + extends DataRecordOptionalFeature[TweetCandidate, Map[String, Double]] + with SparseContinuousDataRecordCompatible { + override val featureName: String = + RecapFeatures.GROK_CATEGORY_SCORES.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredInterests) + } + + object GrokTagsFeature + extends Feature[TweetCandidate, Set[String]] + with SparseBinaryDataRecordCompatible { + override val featureName: String = RecapFeatures.GROK_TAGS.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set( + pd.PersonalDataType.InferredLanguage) + } + + object GrokIsGoreFeature + extends DataRecordOptionalFeature[TweetCandidate, Boolean] + with BoolDataRecordCompatible { + override val featureName: String = + RecapFeatures.GROK_IS_GORE.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty + } + + object GrokIsNsfwFeature + extends DataRecordOptionalFeature[TweetCandidate, Boolean] + with BoolDataRecordCompatible { + override val featureName: String = + RecapFeatures.GROK_IS_NSFW.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty + } + + object GrokIsSoftNsfwFeature + extends DataRecordOptionalFeature[TweetCandidate, Boolean] + with BoolDataRecordCompatible { + override val featureName: String = + RecapFeatures.GROK_IS_SOFT_NSFW.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty + } + + object GrokIsSpamFeature + extends DataRecordOptionalFeature[TweetCandidate, Boolean] + with BoolDataRecordCompatible { + override val featureName: String = + RecapFeatures.GROK_IS_SPAM.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty + } + + object GrokIsViolentFeature + extends DataRecordOptionalFeature[TweetCandidate, Boolean] + with BoolDataRecordCompatible { + override val featureName: String = + RecapFeatures.GROK_IS_VIOLENT.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty + } + + object GrokIsLowQualityFeature + extends DataRecordOptionalFeature[TweetCandidate, Boolean] + with BoolDataRecordCompatible { + override val featureName: String = + RecapFeatures.GROK_IS_LOW_QUALITY.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredInterests) + } + + object GrokIsOcrFeature + extends DataRecordOptionalFeature[TweetCandidate, Boolean] + with BoolDataRecordCompatible { + override val featureName: String = + RecapFeatures.GROK_IS_OCR.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredInterests) + } + + object GrokSunnyScoreFeature + extends DataRecordOptionalFeature[TweetCandidate, Double] + with DoubleDataRecordCompatible { + override val featureName: String = RecapFeatures.GROK_SUNNY_SCORE.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set( + pd.PersonalDataType.EngagementScore) + } + + // Used only for metrics tracking. Does not affect the recommendations. + object GrokPoliticalInclinationFeature + extends Feature[TweetCandidate, Option[hmt.PoliticalInclination]] + + object GrokSlopScoreFeature extends Feature[TweetCandidate, Option[Long]] + + object DedupClusterIdFeature extends Feature[TweetCandidate, Option[Long]] + object DedupClusterId88Feature extends Feature[TweetCandidate, Option[Long]] object RetweetedByCountFeature extends DataRecordFeature[TweetCandidate, Double] with DoubleDataRecordCompatible { @@ -119,19 +249,25 @@ object HomeFeatures { object TopicIdSocialContextFeature extends Feature[TweetCandidate, Option[Long]] object TopicContextFunctionalityTypeFeature extends Feature[TweetCandidate, Option[TopicContextFunctionalityType]] - object FromInNetworkSourceFeature extends Feature[TweetCandidate, Boolean] + object BasketballContextFeature extends Feature[TweetCandidate, Option[urt.BasketballContext]] + object GenericPostContextFeature extends Feature[TweetCandidate, Option[urt.GenericContext]] + object FromInNetworkSourceFeature extends Feature[TweetCandidate, Boolean] object FullScoringSucceededFeature extends Feature[TweetCandidate, Boolean] object HasDisplayedTextFeature extends Feature[TweetCandidate, Boolean] object InReplyToTweetIdFeature extends Feature[TweetCandidate, Option[Long]] object InReplyToUserIdFeature extends Feature[TweetCandidate, Option[Long]] + object IsArticleFeature extends Feature[TweetCandidate, Boolean] object IsAncestorCandidateFeature extends Feature[TweetCandidate, Boolean] + object IsBoostedCandidateFeature extends Feature[TweetCandidate, Boolean] object IsExtendedReplyFeature extends DataRecordFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { override val featureName: String = RecapFeatures.IS_EXTENDED_REPLY.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } + object IsInReplyToReplyOrDirectedFeature extends Feature[TweetCandidate, Boolean] + object IsInReplyToRetweetFeature extends Feature[TweetCandidate, Boolean] object IsRandomTweetFeature extends DataRecordFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { @@ -145,31 +281,46 @@ object HomeFeatures { object LastScoredTimestampMsFeature extends Feature[TweetCandidate, Option[Long]] object NonSelfFavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object NumImagesFeature extends Feature[TweetCandidate, Option[Int]] - object OriginalTweetCreationTimeFromSnowflakeFeature extends Feature[TweetCandidate, Option[Time]] object PositionFeature extends Feature[TweetCandidate, Option[Int]] + // Internal id generated per prediction service request object PredictionRequestIdFeature extends Feature[TweetCandidate, Option[Long]] object QuotedTweetIdFeature extends Feature[TweetCandidate, Option[Long]] object QuotedUserIdFeature extends Feature[TweetCandidate, Option[Long]] + object PhoenixScoreFeature extends Feature[TweetCandidate, Option[Double]] object ScoreFeature extends Feature[TweetCandidate, Option[Double]] + object IsColdStartPostFeature extends Feature[TweetCandidate, Boolean] object SemanticCoreIdFeature extends Feature[TweetCandidate, Option[Long]] - // Key for kafka logging - object ServedIdFeature extends Feature[TweetCandidate, Option[Long]] object SimclustersTweetTopKClustersWithScoresFeature extends Feature[TweetCandidate, Map[String, Double]] - object SocialContextFeature extends Feature[TweetCandidate, Option[tst.SocialContext]] - object SourceTweetIdFeature + // Tweet ID of the source tweet if the candidate is a retweet + object SourceTweetIdFeature extends Feature[TweetCandidate, Option[Long]] + + // Tweet ID of the source tweet if the candidate is a retweet. Tweet id of the candidate otherwise + object OriginalTweetIdFeature extends DataRecordOptionalFeature[TweetCandidate, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = TimelinesSharedFeatures.SOURCE_TWEET_ID.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.TweetId) } object SourceUserIdFeature extends Feature[TweetCandidate, Option[Long]] - object StreamToKafkaFeature extends Feature[TweetCandidate, Boolean] - object SuggestTypeFeature extends Feature[TweetCandidate, Option[st.SuggestType]] + object ServedTypeFeature extends Feature[TweetCandidate, hmt.ServedType] object TSPMetricTagFeature extends Feature[TweetCandidate, Set[tsp.MetricTag]] object TweetLanguageFeature extends Feature[TweetCandidate, Option[String]] + object TweetMediaIdsFeature extends Feature[TweetCandidate, Seq[Long]] + object TweetMediaClusterIdsFeature extends Feature[TweetCandidate, Map[Long, Long]] + object TweetMediaCompletionRateFeature extends Feature[TweetCandidate, Option[Double]] + object ClipImageClusterIdsFeature extends Feature[TweetCandidate, Map[Long, Long]] + object MultiModalEmbeddingsFeature extends Feature[TweetCandidate, Option[Seq[Double]]] + object FirstMediaIdFeature + extends DataRecordOptionalFeature[TweetCandidate, Long] + with LongDiscreteDataRecordCompatible { + override val featureName: String = TimelinesSharedFeatures.FIRST_MEDIA_ID.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.TweetId) + } object TweetUrlsFeature extends Feature[TweetCandidate, Seq[String]] + + object ViewCountFeature extends Feature[TweetCandidate, Option[Long]] object VideoDurationMsFeature extends Feature[TweetCandidate, Option[Int]] object ViewerIdFeature extends DataRecordFeature[TweetCandidate, Long] @@ -182,14 +333,24 @@ object HomeFeatures { object MentionScreenNameFeature extends Feature[TweetCandidate, Seq[String]] object HasImageFeature extends Feature[TweetCandidate, Boolean] object HasVideoFeature extends Feature[TweetCandidate, Boolean] + object VideoAspectRatioFeature extends Feature[TweetCandidate, Option[Float]] + object VideoDisplayTypeFeature extends Feature[TweetCandidate, Option[ModuleDisplayType]] + object VideoHeightFeature extends Feature[TweetCandidate, Option[Short]] + object VideoWidthFeature extends Feature[TweetCandidate, Option[Short]] + object MediaIdFeature extends Feature[TweetCandidate, Option[Long]] + object HasMultipleMedia extends Feature[TweetCandidate, Boolean] + object MediaCategoryFeature extends Feature[TweetCandidate, Option[MediaCategory]] + object SemanticAnnotationIdsFeature extends Feature[TweetCandidate, Seq[Long]] + object TweetTypeMetricsFeature extends Feature[TweetCandidate, Option[Seq[Byte]]] + object CurrentPinnedTweetFeature extends Feature[TweetCandidate, Option[Long]] // Tweetypie VF Features object IsHydratedFeature extends Feature[TweetCandidate, Boolean] - object IsNsfwFeature extends Feature[TweetCandidate, Boolean] + object OonNsfwFeature extends Feature[TweetCandidate, Boolean] object QuotedTweetDroppedFeature extends Feature[TweetCandidate, Boolean] // Raw Tweet Text from Tweetypie object TweetTextFeature extends Feature[TweetCandidate, Option[String]] - + object TweetTextTokensFeature extends Feature[TweetCandidate, Option[Seq[Int]]] object AuthorEnabledPreviewsFeature extends Feature[TweetCandidate, Boolean] object IsTweetPreviewFeature extends Feature[TweetCandidate, Boolean] @@ -211,7 +372,63 @@ object HomeFeatures { override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.ClientType) } object CachedScoredTweetsFeature extends Feature[PipelineQuery, Seq[hmt.ScoredTweet]] - object DeviceLanguageFeature extends Feature[PipelineQuery, Option[String]] + + object FollowsSportsAccountFeature extends Feature[PipelineQuery, Boolean] + + object DeviceCountryFeature + extends DataRecordOptionalFeature[PipelineQuery, String] + with StringDataRecordCompatible { + override def featureName: String = RequestContextFeatures.COUNTRY_CODE.getFeatureName + + override def personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.PrivateCountryOrRegion, pd.PersonalDataType.InferredCountry) + } + + object DeviceLanguageFeature + extends DataRecordOptionalFeature[PipelineQuery, String] + with StringDataRecordCompatible { + override def featureName: String = RequestContextFeatures.LANGUAGE_CODE.getFeatureName + + override def personalDataTypes: Set[pd.PersonalDataType] = Set( + pd.PersonalDataType.GeneralSettings, + pd.PersonalDataType.ProvidedLanguage, + pd.PersonalDataType.InferredLanguage) + } + + object UuaUserGenderFeature + extends DataRecordOptionalFeature[PipelineQuery, String] + with StringDataRecordCompatible { + override def featureName: String = UserHealthFeatures.UserGender.getFeatureName + + override def personalDataTypes: Set[pd.PersonalDataType] = Set( + pd.PersonalDataType.GeneralSettings, + pd.PersonalDataType.ProvidedGender, + pd.PersonalDataType.InferredGender) + } + + object UuaUserStateFeature + extends DataRecordOptionalFeature[PipelineQuery, Long] + with LongDiscreteDataRecordCompatible { + override def featureName: String = UserHealthFeatures.UserState.getFeatureName + + override def personalDataTypes: Set[pd.PersonalDataType] = Set( + pd.PersonalDataType.UserState, + pd.PersonalDataType.UserType + ) + } + + object UuaUserAgeBucketFeature + extends DataRecordOptionalFeature[PipelineQuery, String] + with StringDataRecordCompatible { + override def featureName: String = UserHealthFeatures.UserAgeBucket.getFeatureName + + override def personalDataTypes: Set[pd.PersonalDataType] = Set( + pd.PersonalDataType.GeneralSettings, + pd.PersonalDataType.ProvidedAge, + pd.PersonalDataType.InferredAge + ) + } + object DismissInfoFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Map[st.SuggestType, Option[DismissInfo]]] { override def defaultValue: Map[st.SuggestType, Option[DismissInfo]] = Map.empty @@ -247,68 +464,160 @@ object HomeFeatures { override def featureName: String = SharedFeatures.GUEST_ID.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.GuestId) } + + object GrokAnnotationsFeature extends Feature[TweetCandidate, Option[hmt.GrokAnnotations]] + + object GrokTopCategoryFeature extends Feature[TweetCandidate, Option[Long]] + object HasDarkRequestFeature extends Feature[PipelineQuery, Option[Boolean]] object ImpressionBloomFilterFeature extends FeatureWithDefaultOnFailure[PipelineQuery, blm.ImpressionBloomFilterSeq] { override def defaultValue: blm.ImpressionBloomFilterSeq = blm.ImpressionBloomFilterSeq(Seq.empty) } + object ImpressedMediaIds extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { + override val defaultValue: Seq[Long] = Seq.empty + } object IsForegroundRequestFeature extends Feature[PipelineQuery, Boolean] object IsLaunchRequestFeature extends Feature[PipelineQuery, Boolean] object LastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] + object LastNegativeFeedbackTimeFeature extends Feature[PipelineQuery, Option[Time]] + object LowSignalUserFeature extends Feature[PipelineQuery, Boolean] + object NaviClientConfigFeature extends Feature[PipelineQuery, NaviClientConfig] object NonPollingTimesFeature extends Feature[PipelineQuery, Seq[Long]] object PersistenceEntriesFeature extends Feature[PipelineQuery, Seq[TimelineResponseV3]] object PollingFeature extends Feature[PipelineQuery, Boolean] object PullToRefreshFeature extends Feature[PipelineQuery, Boolean] // Scores from Real Graph representing the relationship between the viewer and another user object RealGraphInNetworkScoresFeature extends Feature[PipelineQuery, Map[UserId, Double]] + object ImmersiveClientEmbeddingsFeature extends Feature[PipelineQuery, Map[Long, Seq[Double]]] object RequestJoinIdFeature extends Feature[TweetCandidate, Option[Long]] - // Internal id generated per request, mainly to deduplicate re-served cached tweets in logging - object ServedRequestIdFeature extends Feature[PipelineQuery, Option[Long]] + // Internal id generated per request + object ServedIdFeature extends Feature[PipelineQuery, Option[Long]] object ServedTweetIdsFeature extends Feature[PipelineQuery, Seq[Long]] + object ServedAuthorIdsFeature extends Feature[PipelineQuery, Map[Long, Seq[Long]]] object ServedTweetPreviewIdsFeature extends Feature[PipelineQuery, Seq[Long]] + + object SignupSourceFeature extends Feature[PipelineQuery, Option[SignupSource]] + object SignupCountryFeature extends Feature[PipelineQuery, Option[String]] + object ViewerAllowsAdsPersonalizationFeature extends Feature[PipelineQuery, Option[Boolean]] + object ViewerAllowsForYouRecommendationsFeature extends Feature[PipelineQuery, Option[Boolean]] + object ViewerAllowsDataSharingFeature extends Feature[PipelineQuery, Option[Boolean]] object TimelineServiceTweetsFeature extends Feature[PipelineQuery, Seq[Long]] + object TLSOriginalTweetsWithAuthorFeature + extends Feature[PipelineQuery, Seq[(Long, Option[Long])]] + + object TLSOriginalTweetsWithConfirmedAuthorFeature + extends Feature[PipelineQuery, Seq[(Long, Long)]] + + object TweetAuthorFollowersFeature extends Feature[PipelineQuery, Map[Long, Option[Long]]] + object TimestampFeature extends DataRecordFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { - override def featureName: String = SharedFeatures.TIMESTAMP.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty + override val featureName: String = SharedFeatures.TIMESTAMP.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object TimestampGMTDowFeature extends DataRecordFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { - override def featureName: String = RequestContextFeatures.TIMESTAMP_GMT_DOW.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty + override val featureName: String = RequestContextFeatures.TIMESTAMP_GMT_DOW.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object TimestampGMTHourFeature extends DataRecordFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { - override def featureName: String = RequestContextFeatures.TIMESTAMP_GMT_HOUR.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty + override val featureName: String = RequestContextFeatures.TIMESTAMP_GMT_HOUR.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty + } + object TweetMixerScoreFeature + extends DataRecordOptionalFeature[TweetCandidate, Double] + with DoubleDataRecordCompatible { + override val featureName: String = TimelinesSharedFeatures.CANDIDATE_SOURCE_SCORE.getFeatureName + override val personalDataTypes: Set[PersonalDataType] = Set(pd.PersonalDataType.EngagementScore) + } + + object SourceSignalFeature extends Feature[TweetCandidate, Option[SourceSignal]] + + object DebugStringFeature extends Feature[TweetCandidate, Option[String]] + + object IsSelfThreadFeature + extends DataRecordFeature[PipelineQuery, Boolean] + with BoolDataRecordCompatible { + override val featureName: String = + conversation_features.ConversationFeatures.IS_SELF_THREAD_TWEET.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty + } + + object TweetLanguageFromTweetypieFeature + extends DataRecordOptionalFeature[TweetCandidate, String] + with StringDataRecordCompatible { + override val featureName: String = + LanguageFeatures.TweetLanguageFromTweetypieFeature.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredLanguage) + } + object TweetLanguageFromLanguageSignalFeature + extends DataRecordOptionalFeature[PipelineQuery, String] + with StringDataRecordCompatible { + override val featureName: String = + LanguageFeatures.TweetLanguageFromLanguageSignalFeature.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set( + pd.PersonalDataType.InferredLanguage) + } + object UserActionsFeature extends Feature[PipelineQuery, Option[UserActionSequence]] + object UserActionsByteArrayFeature extends Feature[PipelineQuery, Option[Array[Byte]]] + object UserActionsSizeFeature extends Feature[PipelineQuery, Option[Int]] + object UserActionsContainsExplicitSignalsFeature extends Feature[PipelineQuery, Boolean] + object UserEngagedLanguagesFeature + extends Feature[PipelineQuery, Set[String]] + with SparseBinaryDataRecordCompatible { + override val featureName: String = LanguageFeatures.EngagedLanguages.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = Set( + pd.PersonalDataType.InferredLanguage) } - object TweetImpressionsFeature extends Feature[PipelineQuery, Seq[imp.TweetImpressionsEntry]] object UserFollowedTopicsCountFeature extends Feature[PipelineQuery, Option[Int]] object UserFollowingCountFeature extends Feature[PipelineQuery, Option[Int]] + object UserFollowersCountFeature extends Feature[PipelineQuery, Option[Int]] + object UserRecentEngagementTweetIdsFeature extends Feature[PipelineQuery, Seq[Long]] + object UserLastExplicitSignalTimeFeature extends Feature[PipelineQuery, Option[Time]] + object UserUnderstandableLanguagesFeature extends Feature[PipelineQuery, Seq[String]] object UserScreenNameFeature extends Feature[PipelineQuery, Option[String]] object UserStateFeature extends Feature[PipelineQuery, Option[um.UserState]] object UserTypeFeature extends Feature[PipelineQuery, Option[gt.UserType]] + object ViewerSafetyLabels extends Feature[PipelineQuery, Option[Seq[String]]] + object ViewerIsRateLimited extends Feature[PipelineQuery, Boolean] + object ViewerHasJobRecommendationsEnabled extends Feature[PipelineQuery, Boolean] + object ViewerHasPremiumTier extends Feature[PipelineQuery, Boolean] + object ViewerHasRecruitingOrganizationRecommendationsEnabled + extends Feature[PipelineQuery, Boolean] object WhoToFollowExcludedUserIdsFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { override def defaultValue = Seq.empty } + object NSFWConsumerScoreFeature extends Feature[PipelineQuery, Double] + object NSFWConsumerFollowerScoreFeature extends Feature[PipelineQuery, Double] + + object CurrentDisplayedGrokTopicFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[(Long, String)]] { + override val defaultValue: Option[(Long, String)] = None + } + // Result Features object ServedSizeFeature extends Feature[PipelineQuery, Option[Int]] - object HasRandomTweetFeature extends Feature[PipelineQuery, Boolean] - object IsRandomTweetAboveFeature extends Feature[TweetCandidate, Boolean] + object UniqueAuthorCountFeature extends Feature[PipelineQuery, Option[Int]] + object MaxSingleAuthorCountFeature extends Feature[PipelineQuery, Option[Int]] + object UniqueCategoryCountFeature extends Feature[PipelineQuery, Option[Int]] + object MaxSingleCategoryCountFeature extends Feature[PipelineQuery, Option[Int]] object ServedInConversationModuleFeature extends Feature[TweetCandidate, Boolean] object ConversationModule2DisplayedTweetsFeature extends Feature[TweetCandidate, Boolean] object ConversationModuleHasGapFeature extends Feature[TweetCandidate, Boolean] object SGSValidLikedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object SGSValidFollowedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object PerspectiveFilteredLikedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object ScreenNamesFeature extends Feature[TweetCandidate, Map[Long, String]] object RealNamesFeature extends Feature[TweetCandidate, Map[Long, String]] + object TweetAgeFeature extends Feature[TweetCandidate, Option[Long]] /** * Features around the focal Tweet for Tweets which should be rendered in convo modules. @@ -322,4 +631,5 @@ object HomeFeatures { object FocalTweetRealNamesFeature extends Feature[TweetCandidate, Option[Map[Long, String]]] object FocalTweetScreenNamesFeature extends Feature[TweetCandidate, Option[Map[Long, String]]] object MediaUnderstandingAnnotationIdsFeature extends Feature[TweetCandidate, Seq[Long]] + object AdTagUrlFeature extends Feature[TweetCandidate, Option[String]] } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeLargeEmbeddingsFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeLargeEmbeddingsFeatures.scala new file mode 100644 index 000000000..4ced0e54d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeLargeEmbeddingsFeatures.scala @@ -0,0 +1,50 @@ +package com.twitter.home_mixer.model + +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object HomeLargeEmbeddingsFeatures { + object AuthorLargeEmbeddingsFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() + } + + object AuthorLargeEmbeddingsKeyFeature extends Feature[TweetCandidate, Seq[Long]] + + object OriginalAuthorLargeEmbeddingsFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() + } + + object OriginalAuthorLargeEmbeddingsKeyFeature extends Feature[TweetCandidate, Seq[Long]] + + object TweetLargeEmbeddingsFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() + } + + object TweetLargeEmbeddingsKeyFeature extends Feature[TweetCandidate, Seq[Long]] + + object OriginalTweetLargeEmbeddingsFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() + } + + object OriginalTweetLargeEmbeddingsKeyFeature extends Feature[TweetCandidate, Seq[Long]] + + object UserLargeEmbeddingsFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() + } + + object UserLargeEmbeddingsKeyFeature extends Feature[PipelineQuery, Seq[Long]] +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/NaviClientConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/NaviClientConfig.scala new file mode 100644 index 000000000..61832e3c0 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/NaviClientConfig.scala @@ -0,0 +1,6 @@ +package com.twitter.home_mixer.model + +case class NaviClientConfig( + clientName: String, + customizedBatchSize: Option[Int], + clusterStr: String) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/NavigationIncludeInstruction.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/NavigationIncludeInstruction.scala new file mode 100644 index 000000000..1c31982b3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/NavigationIncludeInstruction.scala @@ -0,0 +1,41 @@ +package com.twitter.home_mixer.model + +import com.twitter.home_mixer.model.request.DeviceContext.RequestContext +import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.IncludeInstruction +import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.FSParam + +/** + * Include a navigation timeline instruction when we satisfy criteria + */ +case class NavigationIncludeInstruction( + ptrEnableParam: FSParam[Boolean], + coldStartEnableParam: FSParam[Boolean], + warmStartEnableParam: FSParam[Boolean], + navigateEnableParam: FSParam[Boolean], + manualRefreshEnableParam: FSParam[Boolean]) + extends IncludeInstruction[PipelineQuery with HasDeviceContext] { + + override def apply( + query: PipelineQuery with HasDeviceContext, + entries: Seq[TimelineEntry] + ): Boolean = { + val requestContext = query.deviceContext.flatMap(_.requestContextValue) + + val ptrEnabled = + query.params(ptrEnableParam) && requestContext.contains(RequestContext.PullToRefresh) + val coldStartEnabled = + query.params(coldStartEnableParam) && requestContext.contains(RequestContext.Launch) + val warmStartEnabled = + query.params(warmStartEnableParam) && requestContext.contains(RequestContext.Foreground) + val manualRefreshEnabled = + query.params(manualRefreshEnableParam) && requestContext.contains( + RequestContext.ManualRefresh) + val navigateEnabled = + query.params(navigateEnableParam) && requestContext.contains(RequestContext.Navigate) + + ptrEnabled || coldStartEnabled || warmStartEnabled || manualRefreshEnabled || navigateEnabled + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/PhoenixPredictedScoreFeature.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/PhoenixPredictedScoreFeature.scala new file mode 100644 index 000000000..c9b9480bf --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/PhoenixPredictedScoreFeature.scala @@ -0,0 +1,193 @@ +package com.twitter.home_mixer.model + +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelWeights +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseProdInPhoenixParams +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam +import com.x.user_action_sequence.ActionName +import com.x.user_action_sequence.ActionName._ + +sealed trait PhoenixPredictedScoreFeature extends Feature[TweetCandidate, Option[Double]] { + def featureName: String + def modelWeightParam: FSBoundedParam[Double] + def actions: Seq[ActionName] + def isEligible(features: FeatureMap): Boolean = true + def prodScoreFeature: PredictedScoreFeature + def useProdFeatureParam: FSParam[Boolean] + def extractScore(features: FeatureMap, query: PipelineQuery): Option[Double] = { + if (query.params(useProdFeatureParam)) + prodScoreFeature.extractScore(features, query) + else features.getOrElse(this, None).orElse(prodScoreFeature.extractScore(features, query)) + } +} + +object PhoenixPredictedFavoriteScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "fav" + override val modelWeightParam = ModelWeights.FavParam + override val actions = Seq(SERVER_TWEET_FAV) + + override def prodScoreFeature = PredictedFavoriteScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdFavForPhoenixParam +} + +object PhoenixPredictedReplyScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "reply" + override val modelWeightParam = ModelWeights.ReplyParam + override val actions = Seq(SERVER_TWEET_REPLY) + + override def prodScoreFeature = PredictedReplyScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdReplyForPhoenixParam +} + +object PhoenixPredictedRetweetScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "retweet" + override val modelWeightParam = ModelWeights.RetweetParam + override val actions = Seq(SERVER_TWEET_QUOTE, SERVER_TWEET_RETWEET) + + override def prodScoreFeature = PredictedRetweetScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdRetweetForPhoenixParam +} + +object PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature + extends PhoenixPredictedScoreFeature { + override val featureName = "click_engage" + override val modelWeightParam = ModelWeights.GoodClickV1Param + override val actions = Seq(CLIENT_TWEET_PHOTO_EXPAND) + + override def prodScoreFeature = PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdGoodClickV1ForPhoenixParam +} + +object PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "click_dwell" + override val modelWeightParam = ModelWeights.GoodClickV2Param + override val actions = Seq(CLIENT_TWEET_CLICK) + + override def prodScoreFeature = PredictedGoodClickConvoDescUamGt2ScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdGoodClickV2ForPhoenixParam +} + +object PhoenixPredictedGoodProfileClickScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "good_profile_click" + override val modelWeightParam = ModelWeights.GoodProfileClickParam + override val actions = Seq(CLIENT_TWEET_CLICK_PROFILE) + + override def prodScoreFeature = PredictedGoodProfileClickScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdProfileClickForPhoenixParam +} + +object PhoenixPredictedVideoQualityViewScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "vqv" + override val modelWeightParam = ModelWeights.VideoQualityViewParam + override val actions = Seq(CLIENT_TWEET_VIDEO_QUALITY_VIEW) + + override def isEligible(features: FeatureMap): Boolean = { + val isVideoDurationGte10Seconds = + (features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) / 1000.0) >= 10 + val hasVideoFeature = features.getOrElse(HasVideoFeature, false) + hasVideoFeature && isVideoDurationGte10Seconds + } + override def prodScoreFeature = PredictedVideoQualityViewScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdVQVForPhoenixParam +} + +object PhoenixPredictedShareScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "share" + override val modelWeightParam = ModelWeights.ShareParam + override val actions = Seq( + CLIENT_TWEET_SHARE_VIA_COPY_LINK, + CLIENT_TWEET_CLICK_SEND_VIA_DIRECT_MESSAGE, + CLIENT_TWEET_SHARE + ) + + override def prodScoreFeature = PredictedShareScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdShareForPhoenixParam +} + +object PhoenixPredictedDwellScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "dwell" + override val modelWeightParam = ModelWeights.DwellParam + override val actions = Seq(CLIENT_TWEET_RECAP_DWELLED) + + override def isEligible(features: FeatureMap): Boolean = { + val isVideoDurationGte10Seconds = + (features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) / 1000.0) >= 10 + val hasVideoFeature = features.getOrElse(HasVideoFeature, false) + !(hasVideoFeature && isVideoDurationGte10Seconds) + } + override def prodScoreFeature = PredictedDwellScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdDwellForPhoenixParam +} + +object PhoenixPredictedOpenLinkScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "open_link" + override val modelWeightParam = ModelWeights.OpenLinkParam + override val actions = Seq(CLIENT_TWEET_OPEN_LINK) + + // Placeholder prod score feature. This should not be used. + override def prodScoreFeature = PredictedShareScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdOpenLinkForPhoenixParam +} + +object PhoenixPredictedScreenshotScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "screenshot" + override val modelWeightParam = ModelWeights.OpenLinkParam + override val actions = Seq(CLIENT_TWEET_TAKE_SCREENSHOT) + + // Placeholder prod score feature. This should not be used. + override def prodScoreFeature = PredictedShareScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdScreenshotForPhoenixParam +} + +object PhoenixPredictedBookmarkScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "bookmark" + override val modelWeightParam = ModelWeights.BookmarkParam + override val actions = Seq(CLIENT_TWEET_BOOKMARK) + + // Placeholder prod score feature. This should not be used. + override def prodScoreFeature = PredictedBookmarkScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdBookmarkForPhoenixParam +} + +// Negative Engagements +object PhoenixPredictedNegativeFeedbackV2ScoreFeature extends PhoenixPredictedScoreFeature { + override val featureName = "negative_feedback_v2" + override val modelWeightParam = ModelWeights.NegativeFeedbackV2Param + override val actions = Seq( + CLIENT_TWEET_NOT_INTERESTED_IN, + CLIENT_TWEET_BLOCK_AUTHOR, + CLIENT_TWEET_MUTE_AUTHOR, + CLIENT_TWEET_REPORT, + ) + + override def prodScoreFeature = PredictedNegativeFeedbackV2ScoreFeature + override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdNegForPhoenixParam +} + +object PhoenixPredictedScoreFeature { + val PhoenixPredictedScoreFeatures: Seq[PhoenixPredictedScoreFeature] = Seq( + PhoenixPredictedFavoriteScoreFeature, + PhoenixPredictedReplyScoreFeature, + PhoenixPredictedRetweetScoreFeature, + PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, + PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature, + PhoenixPredictedGoodProfileClickScoreFeature, + PhoenixPredictedVideoQualityViewScoreFeature, + PhoenixPredictedShareScoreFeature, + PhoenixPredictedDwellScoreFeature, + PhoenixPredictedOpenLinkScoreFeature, + PhoenixPredictedScreenshotScoreFeature, + PhoenixPredictedBookmarkScoreFeature, + // Negative Engagements + PhoenixPredictedNegativeFeedbackV2ScoreFeature, + ) + + val PhoenixPredictedScoreFeatureSet: Set[PhoenixPredictedScoreFeature] = + PhoenixPredictedScoreFeatures.toSet +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/PredictedScoreFeature.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/PredictedScoreFeature.scala new file mode 100644 index 000000000..a11ab5a18 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/PredictedScoreFeature.scala @@ -0,0 +1,336 @@ +package com.twitter.home_mixer.model + +import com.twitter.dal.personal_data.thriftjava.PersonalDataType +import com.twitter.dal.personal_data.{thriftjava => pd} +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableImmersiveVQV +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTenSecondsLogicForVQV +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ScoreThresholdForVQVParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableBinarySchemeForVQVParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.BinarySchemeConstantForVQVParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ScoreThresholdForDwellParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableDwellOrVQVParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableBinarySchemeForDwellParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelBiases +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelDebiases +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelWeights +import com.twitter.ml.api.DataType +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature +import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.prediction.features.recap.RecapFeatures +import com.twitter.ml.api.thriftscala.GeneralTensor +import com.twitter.product_mixer.core.feature.datarecord.GeneralTensorDataRecordCompatible + +sealed trait PredictedScoreFeature + extends DataRecordOptionalFeature[TweetCandidate, Double] + with DoubleDataRecordCompatible { + + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty + + // Note: Determine whether the score prediction should contribute to the weighted score + // Currently, it is being used for PredictedVideoQualityViewScoreFeature + def isEligible( + features: FeatureMap, + query: PipelineQuery + ): Boolean = true + def statName: String + def modelWeightParam: FSBoundedParam[Double] + def modelBiasParam: Option[FSBoundedParam[Double]] = None + def modelDebiasParam: Option[FSBoundedParam[Double]] = None + def extractScore(features: FeatureMap, query: PipelineQuery): Option[Double] = + features.getOrElse(this, None) + + def weightQueryFeatureName: String + lazy val weightQueryFeature: Feature[PipelineQuery, Option[Double]] = + PredictedScoreFeature.getDataRecordFeatureFromName(weightQueryFeatureName) + def biasQueryFeatureName: Option[String] = None + lazy val biasQueryFeature: Option[Feature[PipelineQuery, Option[Double]]] = + biasQueryFeatureName.map(PredictedScoreFeature.getDataRecordFeatureFromName) + + def debiasQueryFeatureName: Option[String] = None + + lazy val debiasQueryFeature: Option[Feature[PipelineQuery, Option[GeneralTensor]]] = + debiasQueryFeatureName.map(PredictedScoreFeature.getGeneralTensorFeatureFromName) +} + +object PredictedFavoriteScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_FAVORITED.getFeatureName + override val statName = "fav" + override val modelWeightParam = ModelWeights.FavParam + override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_FAVORITED.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.FavParam) + override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_FAVORITED.getFeatureName) +} + +object PredictedReplyScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_REPLIED.getFeatureName + override val statName = "reply" + override val modelWeightParam = ModelWeights.ReplyParam + override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_REPLIED.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.ReplyParam) + override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_REPLIED.getFeatureName) +} + +object PredictedRetweetScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_RETWEETED.getFeatureName + override val statName = "retweet" + override val modelWeightParam = ModelWeights.RetweetParam + override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_RETWEETED.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.RetweetParam) + override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_RETWEETED.getFeatureName) +} + +object PredictedReplyEngagedByAuthorScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_REPLIED_REPLY_ENGAGED_BY_AUTHOR.getFeatureName + override val statName = "reply_engaged_by_author" + override val modelWeightParam = ModelWeights.ReplyEngagedByAuthorParam + override val weightQueryFeatureName = + RecapFeatures.WEIGHT_IS_REPLIED_REPLY_ENGAGED_BY_AUTHOR.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.ReplyEngagedByAuthorParam) + override val debiasQueryFeatureName = + Some(RecapFeatures.DEBIAS_IS_REPLIED_REPLY_ENGAGED_BY_AUTHOR.getFeatureName) +} + +object PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V1.getFeatureName + override val statName = "click_engaged" // click_convo_desc_favorited_or_replied + override val modelWeightParam = ModelWeights.GoodClickV1Param + override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_GOOD_CLICKED_V1.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.GoodClickV1Param) + override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_GOOD_CLICKED_V1.getFeatureName) +} + +object PredictedGoodClickConvoDescUamGt2ScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V2.getFeatureName + override val statName = "click_dwell" // good_click_convo_desc_uam_gt_2 + override val modelWeightParam = ModelWeights.GoodClickV2Param + override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_GOOD_CLICKED_V2.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.GoodClickV2Param) + override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_GOOD_CLICKED_V2.getFeatureName) +} + +object PredictedGoodProfileClickScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_PROFILE_CLICKED_AND_PROFILE_ENGAGED.getFeatureName + override val statName = "good_profile_click" + override val modelWeightParam = ModelWeights.GoodProfileClickParam + override val weightQueryFeatureName = + RecapFeatures.WEIGHT_IS_PROFILE_CLICKED_AND_PROFILE_ENGAGED.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.GoodProfileClickParam) + override val debiasQueryFeatureName = + Some(RecapFeatures.DEBIAS_IS_PROFILE_CLICKED_AND_PROFILE_ENGAGED.getFeatureName) +} + +object PredictedVideoQualityViewImmersiveScoreFeature extends PredictedScoreFeature { + override def isEligible( + features: FeatureMap, + query: PipelineQuery + ): Boolean = { + query.params(EnableImmersiveVQV) + } + override val featureName: String = + RecapFeatures.PREDICTED_IS_VIDEO_QUALITY_VIEWED_IMMERSIVE.getFeatureName + override val statName = "vqv_immersive" // video_quality_viewed_immersive + override val modelWeightParam = ModelWeights.VideoQualityViewImmersiveParam + override val weightQueryFeatureName = + RecapFeatures.WEIGHT_IS_VIDEO_QUALITY_VIEWED_IMMERSIVE.getFeatureName + override val modelBiasParam = Some(ModelBiases.VideoQualityViewImmersiveParam) + override val biasQueryFeatureName = Some( + RecapFeatures.BIAS_IS_VIDEO_QUALITY_VIEWED_IMMERSIVE.getFeatureName) + override val modelDebiasParam = Some(ModelDebiases.VideoQualityViewImmersiveParam) + override val debiasQueryFeatureName = + Some(RecapFeatures.DEBIAS_IS_VIDEO_QUALITY_VIEWED_IMMERSIVE.getFeatureName) + +} + +object PredictedVideoQualityViewScoreFeature extends PredictedScoreFeature { + override def isEligible( + features: FeatureMap, + query: PipelineQuery + ): Boolean = { + val isTenSecondsLogicEnabled = query.params(EnableTenSecondsLogicForVQV) + val isVideoDurationGte10Seconds = + (features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) / 1000.0) >= 10 + val hasVideoFeature = features.getOrElse(HasVideoFeature, false) + + hasVideoFeature && (!isTenSecondsLogicEnabled || isVideoDurationGte10Seconds) + } + override val featureName: String = RecapFeatures.PREDICTED_IS_VIDEO_QUALITY_VIEWED.getFeatureName + override val statName = "vqv" // video_quality_viewed + override val modelWeightParam = ModelWeights.VideoQualityViewParam + override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_VIDEO_QUALITY_VIEWED.getFeatureName + override val modelBiasParam = Some(ModelBiases.VideoQualityViewParam) + override val biasQueryFeatureName = Some( + RecapFeatures.BIAS_IS_VIDEO_QUALITY_VIEWED.getFeatureName) + override val modelDebiasParam = Some(ModelDebiases.VideoQualityViewParam) + override val debiasQueryFeatureName = Some( + RecapFeatures.DEBIAS_IS_VIDEO_QUALITY_VIEWED.getFeatureName) + + override def extractScore(features: FeatureMap, query: PipelineQuery): Some[Double] = { + // For VQV, if the score is below a threshold, we return 0 + val vqvScore = features.getOrElse(this, None).getOrElse(0.0) + if (vqvScore < query.params(ScoreThresholdForVQVParam)) { + // the default threshold is 0.0, vqvScore should be always non-negative + Some(0.0) + } else if (query.params(EnableBinarySchemeForVQVParam)) { + // If the binary scheme is enabled, we return a constant + Some(query.params(BinarySchemeConstantForVQVParam)) + } else { + Some(vqvScore) + } + } + +} + +object PredictedBookmarkScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_BOOKMARKED.getFeatureName + override val statName = "bookmark" + override val modelWeightParam = ModelWeights.BookmarkParam + override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_BOOKMARKED.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.BookmarkParam) + override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_BOOKMARKED.getFeatureName) +} + +object PredictedShareScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_SHARED.getFeatureName + override val statName = "share" + override val modelWeightParam = ModelWeights.ShareParam + override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_SHARED.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.ShareParam) + override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_SHARED.getFeatureName) +} + +object PredictedDwellScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_DWELLED.getFeatureName + override val statName = "dwell" + override val modelWeightParam = ModelWeights.DwellParam + override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_DWELLED.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.DwellParam) + override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_DWELLED.getFeatureName) + + override def isEligible( + features: FeatureMap, + query: PipelineQuery + ): Boolean = { + val isTenSecondsLogicEnabled = query.params(EnableTenSecondsLogicForVQV) + val isVideoDurationGte10Seconds = + (features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) / 1000.0) >= 10 + val hasVideoFeature = features.getOrElse(HasVideoFeature, false) + + val isEligibleForVqv = + hasVideoFeature && (!isTenSecondsLogicEnabled || isVideoDurationGte10Seconds) + !(query.params(EnableDwellOrVQVParam) && isEligibleForVqv) + } + + override def extractScore(features: FeatureMap, query: PipelineQuery): Some[Double] = { + // For Dwell, if the score is below a threshold, we return 0 + val dwellScore = features.getOrElse(this, None).getOrElse(0.0) + if (dwellScore < query.params(ScoreThresholdForDwellParam)) { + // the default threshold is 0.0, dwellScore should be always non-negative + Some(0.0) + } else if (query.params(EnableBinarySchemeForDwellParam)) { + // If the binary scheme is enabled, we return a constant + Some(query.params(ScoreThresholdForDwellParam)) + } else { + Some(dwellScore) + } + } +} + +object PredictedVideoWatchTimeScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_VIDEO_WATCH_TIME_MS.getFeatureName + override val statName = "video_watch_time_ms" + override val modelWeightParam = ModelWeights.VideoWatchTimeMsParam + override val weightQueryFeatureName = RecapFeatures.WEIGHT_VIDEO_WATCH_TIME_MS.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.VideoWatchTimeMsParam) + override val debiasQueryFeatureName = Some( + RecapFeatures.DEBIAS_VIDEO_WATCH_TIME_MS.getFeatureName) +} + +// Negative Engagements +object PredictedNegativeFeedbackV2ScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_NEGATIVE_FEEDBACK_V2.getFeatureName + override val statName = "negative_feedback_v2" + override val modelWeightParam = ModelWeights.NegativeFeedbackV2Param + override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_NEGATIVE_FEEDBACK_V2.getFeatureName + override val modelDebiasParam = Some(ModelDebiases.NegativeFeedbackV2Param) + override val debiasQueryFeatureName = Some( + RecapFeatures.DEBIAS_IS_NEGATIVE_FEEDBACK_V2.getFeatureName) +} + +object PredictedVideoQualityWatchScoreFeature extends PredictedScoreFeature { + override def isEligible( + features: FeatureMap, + query: PipelineQuery + ) = { + features.getOrElse(HasVideoFeature, false) && (features + .getOrElse(VideoDurationMsFeature, None).getOrElse(0) / 1000.0) >= 10 + } + override val featureName: String = + RecapFeatures.PREDICTED_IS_VIDEO_QUALITY_WATCH.getFeatureName + override val statName = "video_quality_watched" + override val modelWeightParam = ModelWeights.VideoQualityWatchParam + override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_VIDEO_QUALITY_WATCHED.getFeatureName + override val modelBiasParam = Some(ModelBiases.VideoQualityWatchParam) + override val biasQueryFeatureName = Some( + RecapFeatures.BIAS_IS_VIDEO_QUALITY_WATCHED.getFeatureName) + override val modelDebiasParam = Some(ModelDebiases.VideoQualityWatchParam) + override val debiasQueryFeatureName = Some( + RecapFeatures.DEBIAS_IS_VIDEO_QUALITY_WATCHED.getFeatureName) +} + +object PredictedScoreFeature { + val PredictedScoreFeatures: Seq[PredictedScoreFeature] = Seq( + PredictedFavoriteScoreFeature, + PredictedReplyScoreFeature, + PredictedRetweetScoreFeature, + PredictedReplyEngagedByAuthorScoreFeature, + PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, + PredictedGoodClickConvoDescUamGt2ScoreFeature, + PredictedGoodProfileClickScoreFeature, + PredictedVideoQualityViewScoreFeature, + PredictedVideoQualityViewImmersiveScoreFeature, + PredictedBookmarkScoreFeature, + PredictedShareScoreFeature, + PredictedDwellScoreFeature, + PredictedVideoQualityWatchScoreFeature, + PredictedVideoWatchTimeScoreFeature, + // Negative Engagements + PredictedNegativeFeedbackV2ScoreFeature, + ) + + val PredictedScoreFeatureSet: Set[PredictedScoreFeature] = PredictedScoreFeatures.toSet + + def getGeneralTensorFeatureFromName( + name: String + ): Feature[PipelineQuery, Option[GeneralTensor]] = { + new DataRecordOptionalFeature[PipelineQuery, GeneralTensor] + with GeneralTensorDataRecordCompatible { + override val featureName: String = name + override val personalDataTypes: Set[PersonalDataType] = Set.empty + override def toString: String = name + override def dataType: DataType = DataType.DOUBLE + } + } + + def getDataRecordFeatureFromName( + name: String + ): Feature[PipelineQuery, Option[Double]] = { + new DataRecordOptionalFeature[PipelineQuery, Double] with DoubleDataRecordCompatible { + override val featureName: String = name + override val personalDataTypes: Set[PersonalDataType] = Set.empty + override def toString: String = name + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source/BUILD.bazel new file mode 100644 index 000000000..3c10a1382 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source/BUILD.bazel @@ -0,0 +1,8 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = ["user-signal-service/thrift/src/main/thrift:thrift-scala"], + exports = [], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source/SourceSignal.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source/SourceSignal.scala new file mode 100644 index 000000000..ea7fc9a9b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source/SourceSignal.scala @@ -0,0 +1,9 @@ +package com.twitter.home_mixer.model.candidate_source + +import com.twitter.usersignalservice.{thriftscala => se} + +case class SourceSignal( + id: Long, + signalType: Option[String], + signalEntity: Option[se.SignalEntity], + authorId: Option[Long]) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel index 3883e454e..72fbb770f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel @@ -4,12 +4,14 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup", + "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", "timelineservice/common:model", ], exports = [ + "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.scala index 818380946..2e4fe4f96 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.scala @@ -4,5 +4,6 @@ import com.twitter.product_mixer.core.model.marshalling.request.DebugOptions import com.twitter.util.Time case class HomeMixerDebugOptions( - override val requestTimeOverride: Option[Time]) + override val requestTimeOverride: Option[Time], + override val showIntermediateLogs: Option[Boolean]) extends DebugOptions diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala index 7c27c50d5..3e0ec2ec1 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala @@ -24,17 +24,17 @@ case object ScoredTweetsProduct extends Product { override val stringCenterProject: Option[String] = Some("timelinemixer") } -case object ListTweetsProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("ListTweets") +case object ScoredVideoTweetsProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("ScoredVideoTweets") override val stringCenterProject: Option[String] = Some("timelinemixer") } -case object ListRecommendedUsersProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("ListRecommendedUsers") +case object SubscribedProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("Subscribed") override val stringCenterProject: Option[String] = Some("timelinemixer") } -case object SubscribedProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("Subscribed") +case object HeavyRankerScoresProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("HeavyRankerScores") override val stringCenterProject: Option[String] = Some("timelinemixer") } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala index dddd733f3..0b9b14bbd 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala @@ -1,6 +1,8 @@ package com.twitter.home_mixer.model.request import com.twitter.dspbidder.commons.thriftscala.DspClientContext +import com.twitter.home_mixer.model.signup.SignupSource +import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.model.marshalling.request.ProductContext case class FollowingProductContext( @@ -12,31 +14,36 @@ case class FollowingProductContext( case class ForYouProductContext( deviceContext: Option[DeviceContext], seenTweetIds: Option[Seq[Long]], - dspClientContext: Option[DspClientContext], - pushToHomeTweetId: Option[Long]) + dspClientContext: Option[DspClientContext]) extends ProductContext case class ScoredTweetsProductContext( deviceContext: Option[DeviceContext], seenTweetIds: Option[Seq[Long]], servedTweetIds: Option[Seq[Long]], - backfillTweetIds: Option[Seq[Long]]) + backfillTweetIds: Option[Seq[Long]], + signupCountryCode: Option[String], + allowForYouRecommendations: Option[Boolean], + signupSource: Option[SignupSource], + followerCount: Option[Int], + servedAuthorIds: Option[Map[Long, Seq[Long]]] = None) extends ProductContext -case class ListTweetsProductContext( - listId: Long, +case class ScoredVideoTweetsProductContext( deviceContext: Option[DeviceContext], - dspClientContext: Option[DspClientContext]) - extends ProductContext - -case class ListRecommendedUsersProductContext( - listId: Long, - selectedUserIds: Option[Seq[Long]], - excludedUserIds: Option[Seq[Long]], - listName: Option[String]) + seenTweetIds: Option[Seq[Long]], + videoType: Option[t.VideoType], + pinnedRelatedTweetIds: Option[Seq[Long]], + scorePinnedTweetsOnly: Option[Boolean], + immersiveClientMetadata: Option[t.ImmersiveClientMetadata]) extends ProductContext case class SubscribedProductContext( deviceContext: Option[DeviceContext], seenTweetIds: Option[Seq[Long]]) extends ProductContext + +case class HeavyRankerScoresProductContext( + deviceContext: Option[DeviceContext], + tweetIds: Option[Seq[Long]]) + extends ProductContext diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup/BUILD.bazel new file mode 100644 index 000000000..54e356a77 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup/BUILD.bazel @@ -0,0 +1,8 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup/SignupSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup/SignupSource.scala new file mode 100644 index 000000000..782348c29 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup/SignupSource.scala @@ -0,0 +1,6 @@ +package com.twitter.home_mixer.model.signup + +sealed trait SignupSource + +case object Onboard extends SignupSource +case object MarchMadness extends SignupSource diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel index b7fbca0d3..2c0f4f12e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel @@ -5,84 +5,84 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/com/twitter/bijection:thrift", - "3rdparty/jvm/com/twitter/src/java/com/twitter/logpipeline/client:logpipeline-event-publisher-thin", "3rdparty/jvm/com/twitter/storehaus:core", - "3rdparty/jvm/io/netty:netty4-tcnative-boringssl-static", + "3rdparty/jvm/io/grpc:grpc-netty", + "cproxy/thrift/src/main/thrift:thrift-scala", + "deferredrpc/client/src/main/scala", + "deferredrpc/client/src/main/thrift:thrift-scala", + "escherbird/src/scala/com/twitter/escherbird/util/uttclient", + "escherbird/src/thrift/com/twitter/escherbird/utt:strato-columns-scala", "eventbus/client/src/main/scala/com/twitter/eventbus/client", + "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", "finagle-internal/finagle-grpc/src/main/scala", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client", - "finagle/finagle-core/src/main", - "finagle/finagle-memcached/src/main/scala", - "finagle/finagle-mux/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests:package", + "graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala", + "home-mixer-features/thrift/src/main/thrift:thrift-java", + "home-mixer-features/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/store", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", "home-mixer/thrift/src/main/thrift:thrift-scala", "interests-service/thrift/src/main/thrift:thrift-scala", + "kafka/finagle-kafka/finatra-kafka/src/main/scala", + "limiter/thrift-only/src/main/thrift:thrift-scala", + "media-understanding/video-summary/thrift/src/main/thrift:thrift-scala", "people-discovery/api/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client", "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client", "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client", - "servo/client/src/main/scala/com/twitter/servo/client", "servo/manhattan", - "servo/util", - "socialgraph/server/src/main/scala/com/twitter/socialgraph/util", + "src/java/com/twitter/logpipeline/client:event-publisher-client-lib", + "src/java/com/twitter/logpipeline/client/common:event-publisher-core", + "src/java/com/twitter/logpipeline/client/serializers:event-publisher-serializers", "src/scala/com/twitter/ml/featurestore/lib", "src/scala/com/twitter/scalding_internal/multiformat/format", + "src/scala/com/twitter/simclusters_v2/common", "src/scala/com/twitter/storehaus_internal", + "src/scala/com/twitter/storehaus_internal/offline", "src/scala/com/twitter/summingbird_internal/bijection:bijection-implicits", + "src/scala/com/twitter/summingbird_internal/runner/store_config", "src/scala/com/twitter/timelines/util", + "src/scala/com/twitter/wtf/entity_real_graph/summingbird/client", + "src/scala/com/twitter/wtf/entity_real_graph/summingbird/common/config", "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", "src/thrift/com/twitter/clientapp/gen:clientapp-scala", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", "src/thrift/com/twitter/manhattan:v1-scala", "src/thrift/com/twitter/manhattan:v2-scala", - "src/thrift/com/twitter/onboarding/relevance/features:features-java", - "src/thrift/com/twitter/search:blender-scala", + "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/service/metastore/gen:thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/timelines/author_features:thrift-java", - "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "src/thrift/com/twitter/timelines/impression_store:thrift-scala", - "src/thrift/com/twitter/timelines/real_graph:real_graph-scala", + "src/thrift/com/twitter/timelines/realtime_aggregates:thrift-scala", + "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/topic_recos:topic_recos-thrift-java", "src/thrift/com/twitter/user_session_store:thrift-java", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "stitch/stitch-socialgraph", "stitch/stitch-tweetypie", - "strato/src/main/scala/com/twitter/strato/client", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", + "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/rankedtweetcaching", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelines:decider", + "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/scored_candidate_features_cache", "timelines/src/main/scala/com/twitter/timelines/clients/ads", "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", + "timelines/src/main/scala/com/twitter/timelines/clients/memcache_common", "timelines/src/main/scala/com/twitter/timelines/clients/predictionservice", - "timelines/src/main/scala/com/twitter/timelines/clients/strato", "timelines/src/main/scala/com/twitter/timelines/clients/strato/topics", "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/store", "timelines/src/main/scala/com/twitter/timelines/util/stats", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", + "topic-social-proof/server/src/main/scala/com/twitter/tsp/stores", + "topiclisting/common/src/main/scala/com/twitter/topiclisting/clients/utt", + "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", + "topiclisting/topiclisting-utt/src/main/scala/com/twitter/topiclisting/utt", "tweetconvosvc/client/src/main/scala/com/twitter/tweetconvosvc/client/builder", - "twitter-config/yaml", + "user_history_transformer/protobuf/src/main/protobuf/com/x/user_action_sequence", + "user_history_transformer/service/src/main/java/com/x/user_action_sequence", ], exports = [ "timelines/src/main/scala/com/twitter/timelines/clients/predictionservice", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClusterDetailsModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClusterDetailsModule.scala new file mode 100644 index 000000000..12ac7666a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClusterDetailsModule.scala @@ -0,0 +1,45 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.bijection.Bufferable +import com.twitter.bijection.Injection +import com.twitter.bijection.scrooge.CompactScalaCodec +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.inject.TwitterModule +import com.twitter.simclusters_v2.thriftscala.ClusterDetails +import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams +import com.twitter.storehaus.ReadableStore +import com.twitter.storehaus_internal.manhattan.Athena +import com.twitter.storehaus_internal.manhattan.ManhattanRO +import com.twitter.storehaus_internal.manhattan.ManhattanROConfig +import com.twitter.storehaus_internal.util.ApplicationID +import com.twitter.storehaus_internal.util.DatasetName +import com.twitter.storehaus_internal.util.HDFSPath +import javax.inject.Singleton + +object ClusterDetailsModule extends TwitterModule { + + @Provides + @Singleton + def providesClusterDetailsStore( + serviceIdentifier: ServiceIdentifier + ): ReadableStore[String, ClusterDetails] = { + implicit val keyInjection: Injection[(String, Int), Array[Byte]] = + Bufferable.injectionOf[(String, Int)] + implicit val valueInjection: Injection[ClusterDetails, Array[Byte]] = + CompactScalaCodec(ClusterDetails) + + val modelName = "20M_145K_2020" + + ManhattanRO + .getReadableStoreWithMtls[(String, Int), ClusterDetails]( + ManhattanROConfig( + HDFSPath(""), + ApplicationID("simclusters_v2"), + DatasetName("simclusters_v2_cluster_details_20m_145k_2020"), + Athena + ), + ManhattanKVClientMtlsParams(serviceIdentifier) + ).composeKeyMapping(clusterIdString => (modelName, clusterIdString.toInt)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/EarlybirdRealtimeCGModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/EarlybirdRealtimeCGModule.scala new file mode 100644 index 000000000..6a688fcbb --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/EarlybirdRealtimeCGModule.scala @@ -0,0 +1,48 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ +import com.twitter.conversions.PercentOps._ +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRealtimCGEndpoint +import com.twitter.inject.TwitterModule +import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder +import com.twitter.product_mixer.shared_library.thrift_client.Idempotent +import com.twitter.search.earlybird.{thriftscala => t} +import javax.inject.Named +import javax.inject.Singleton +import org.apache.thrift.protocol.TCompactProtocol + +object EarlybirdRealtimeCGModule extends TwitterModule { + + val Label: String = "earlybird-rootrealtimecg" + val Dest: String = "/s/earlybird-rootrealtimecg/root-realtime_cg" + + @Provides + @Singleton + @Named(EarlybirdRealtimCGEndpoint) + def providesEarlybirdRealtimeCGService( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + statsReceiver: StatsReceiver + ): t.EarlybirdService.MethodPerEndpoint = { + + FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ + t.EarlybirdService.ServicePerEndpoint, + t.EarlybirdService.MethodPerEndpoint + ]( + serviceIdentifier = serviceIdentifier, + clientId = clientId, + dest = Dest, + label = Label, + statsReceiver = statsReceiver, + protocolFactoryOverride = Some(new TCompactProtocol.Factory), + idempotency = Idempotent(1.percent), + timeoutPerRequest = 600.milliseconds, + timeoutTotal = 650.milliseconds, + acquisitionTimeout = 1.seconds + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/EventsRecosClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/EventsRecosClientModule.scala new file mode 100644 index 000000000..49e8c0b17 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/EventsRecosClientModule.scala @@ -0,0 +1,30 @@ +package com.twitter.home_mixer.module + +import com.twitter.conversions.DurationOps._ +import com.twitter.events.recos.thriftscala.EventsRecosService +import com.twitter.finagle.thriftmux.MethodBuilder +import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient +import com.twitter.inject.Injector +import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule +import com.twitter.util.Duration + +object EventsRecosClientModule + extends ThriftMethodBuilderClientModule[ + EventsRecosService.ServicePerEndpoint, + EventsRecosService.MethodPerEndpoint + ] + with MtlsClient { + + override val label: String = "events-recos" + override val dest: String = "/s/events-recos/events-recos-service" + override protected def sessionAcquisitionTimeout: Duration = 500.milliseconds + + override protected def configureMethodBuilder( + injector: Injector, + methodBuilder: MethodBuilder + ): MethodBuilder = { + methodBuilder + .withTimeoutPerRequest(450.millis) + .withTimeoutTotal(450.millis) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/GizmoduckTimelinesCacheClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/GizmoduckTimelinesCacheClientModule.scala new file mode 100644 index 000000000..bc6c8dc87 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/GizmoduckTimelinesCacheClientModule.scala @@ -0,0 +1,70 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps.RichDuration +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.gizmoduck.{thriftscala => gt} +import com.twitter.home_mixer.param.HomeMixerInjectionNames.GizmoduckTimelinesCache +import com.twitter.inject.TwitterModule +import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder +import com.twitter.servo.cache.FinagleMemcache +import com.twitter.servo.cache.KeyValueTransformingTtlCache +import com.twitter.servo.cache.ObservableTtlCache +import com.twitter.servo.cache.Serializer +import com.twitter.servo.cache.ThriftSerializer +import com.twitter.servo.cache.TtlCache +import javax.inject.Named +import javax.inject.Singleton +import org.apache.thrift.protocol.TCompactProtocol +import com.twitter.finagle.memcached.compressing.scheme.Lz4 + +object GizmoduckTimelinesCacheClientModule extends TwitterModule { + + private val ScopeName = "GizmoduckTimelinesCache" + private val ProdDest = "/srv#/prod/local/cache/timelines_gizmoduck_secure:twemcaches" + + private val userSerializer: Serializer[gt.User] = { + new ThriftSerializer[gt.User](gt.User, new TCompactProtocol.Factory()) + } + + @Provides + @Singleton + @Named(GizmoduckTimelinesCache) + def providesGizmoduckTimelinesCache( + statsReceiver: StatsReceiver, + serviceIdentifier: ServiceIdentifier + ): TtlCache[Long, gt.User] = { + val memCacheClient = MemcachedClientBuilder.buildMemcachedClient( + destName = ProdDest, + numTries = 1, + numConnections = 1, + requestTimeout = 100.milliseconds, + globalTimeout = 100.milliseconds, + connectTimeout = 100.milliseconds, + acquisitionTimeout = 100.milliseconds, + serviceIdentifier = serviceIdentifier, + statsReceiver = statsReceiver, + compressionScheme = Lz4 + ) + mkCache(new FinagleMemcache(memCacheClient), statsReceiver) + } + + private def mkCache( + finagleMemcache: FinagleMemcache, + statsReceiver: StatsReceiver + ): TtlCache[Long, gt.User] = { + val baseCache: KeyValueTransformingTtlCache[Long, String, gt.User, Array[Byte]] = + new KeyValueTransformingTtlCache( + underlyingCache = finagleMemcache, + transformer = userSerializer, + underlyingKey = { key: Long => key.toString } + ) + ObservableTtlCache( + underlyingCache = baseCache, + statsReceiver = statsReceiver.scope(ScopeName), + windowSize = 1000, + name = ScopeName + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFeaturesModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFeaturesModule.scala new file mode 100644 index 000000000..f06e0dc64 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFeaturesModule.scala @@ -0,0 +1,82 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ +import com.twitter.conversions.PercentOps._ +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.finagle.thrift.RichClientParam +import com.twitter.inject.TwitterModule +import com.twitter.home_mixer_features.{thriftjava => tj} +import com.twitter.home_mixer_features.{thriftscala => ts} +import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder +import com.twitter.product_mixer.shared_library.thrift_client.Idempotent +import javax.inject.Singleton + +object HomeMixerFeaturesModule extends TwitterModule { + + val Label: String = "home-mixer-features" + val Dest: String = "/s/home-mixer/home-mixer-features" + + @Provides + @Singleton + def providesHomeMixerFeaturesService( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + statsReceiver: StatsReceiver, + ): tj.HomeMixerFeatures.ServiceToClient = { + buildClient(serviceIdentifier, clientId, statsReceiver, Dest, Label) + } + + @Provides + @Singleton + def providesHomeMixerFeaturesScalaService( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + statsReceiver: StatsReceiver, + ): ts.HomeMixerFeatures.MethodPerEndpoint = { + FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ + ts.HomeMixerFeatures.ServicePerEndpoint, + ts.HomeMixerFeatures.MethodPerEndpoint + ]( + serviceIdentifier = serviceIdentifier, + clientId = clientId, + dest = Dest, + label = Label, + statsReceiver = statsReceiver, + idempotency = Idempotent(1.percent), + timeoutPerRequest = 300.milliseconds, + timeoutTotal = 300.milliseconds, + acquisitionTimeout = 1.seconds + ) + } + + private def buildClient( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + statsReceiver: StatsReceiver, + dest: String, + label: String + ): tj.HomeMixerFeatures.ServiceToClient = { + val stats = statsReceiver.scope("clnt") + val thriftClient = ThriftMux.client + .withMutualTls(serviceIdentifier) + .withClientId(clientId) + .withLabel(label) + .withStatsReceiver(stats) + .withRequestTimeout(300.milliseconds) + .withSession.acquisitionTimeout(1.second) + .methodBuilder(dest) + .withTimeoutTotal(300.milliseconds) + .idempotent(1.percent) + .newService + + new tj.HomeMixerFeatures.ServiceToClient( + thriftClient, + RichClientParam() + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala index e31d2d9bc..06fbbbf99 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala @@ -28,14 +28,14 @@ object HomeMixerFlagsModule extends TwitterModule { ) flag[Boolean]( - name = ScribeServedCommonFeaturesAndCandidateFeaturesFlag, + name = ScribeFeaturesFlag, default = false, help = "Toggles logging served common features and candidates features to Scribe" ) flag[String]( name = DataRecordMetadataStoreConfigsYmlFlag, - default = "", + default = "mysql_timelines_ro_prod.yml", help = "The YML file that contains the necessary info for creating metadata store MySQL client." ) @@ -45,12 +45,6 @@ object HomeMixerFlagsModule extends TwitterModule { help = "Dark traffic filter decider key" ) - flag[Duration]( - TargetFetchLatency, - 300.millis, - "Target fetch latency from candidate sources for Quality Factor" - ) - flag[Duration]( TargetScoringLatency, 700.millis, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.scala index b68e5b105..5fbe6286b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.scala @@ -1,5 +1,32 @@ package com.twitter.home_mixer.module +import com.google.inject.Provides +import com.twitter.config.yaml.YamlMap import com.twitter.inject.TwitterModule +import javax.inject.Singleton -object HomeMixerResourcesModule extends TwitterModule {} +case class ViralContentCreatorsConfig(creators: Set[Long]) +case class SupportAccountsConfig(accounts: Set[Long]) + +object HomeMixerResourcesModule extends TwitterModule { + + private val ConfigFilePath = "/config/ids.yml" + private val SupportAccountsKey = "support_accounts" + private val ViralContentCreatorsKey = "viral_content_creators" + + private val yaml: YamlMap = YamlMap.load(ConfigFilePath) + + @Singleton + @Provides + def providesViralContentCreatorsConfig: ViralContentCreatorsConfig = { + val contentCreators = yaml.longSeq(ViralContentCreatorsKey).toSet + ViralContentCreatorsConfig(contentCreators) + } + + @Singleton + @Provides + def providesSupportAccountsConfig: SupportAccountsConfig = { + val supportAccounts = yaml.longSeq(SupportAccountsKey).toSet + SupportAccountsConfig(supportAccounts) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InMemoryCacheModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InMemoryCacheModule.scala new file mode 100644 index 000000000..18eab0d9b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InMemoryCacheModule.scala @@ -0,0 +1,64 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.home_mixer.param.HomeMixerInjectionNames.ImageClipClusterIdInMemCache +import com.twitter.home_mixer.param.HomeMixerInjectionNames.IsColdStartPostInMemCache +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClipClusterIdInMemCache +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaCompletionRateInMemCache +import com.twitter.inject.TwitterModule +import com.twitter.servo.cache.ExpiringLruInProcessCache +import com.twitter.servo.cache.InProcessCache +import javax.inject.Named +import javax.inject.Singleton +import scala.util.Random + +object InMemoryCacheModule extends TwitterModule { + @Singleton + @Provides + @Named(MediaClipClusterIdInMemCache) + def providesMediaClipClusterIdInMemCache( + ): InProcessCache[Long, Option[Option[Long]]] = { + val BaseTTL = 4 + val TTL = (BaseTTL + Random.nextInt(3)).minutes + val cache: InProcessCache[Long, Option[Option[Long]]] = + new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 500000) + cache + } + + @Singleton + @Provides + @Named(ImageClipClusterIdInMemCache) + def providesImageClipClusterIdInMemCache( + ): InProcessCache[Long, Option[Option[Long]]] = { + val BaseTTL = 4 + val TTL = (BaseTTL + Random.nextInt(3)).minutes + val cache: InProcessCache[Long, Option[Option[Long]]] = + new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 500000) + cache + } + + @Singleton + @Provides + @Named(MediaCompletionRateInMemCache) + def providesMediaCompletionRateInMemCache( + ): InProcessCache[Long, Double] = { + val BaseTTL = 20 + val TTL = (BaseTTL + Random.nextInt(15)).minutes + val cache: InProcessCache[Long, Double] = + new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 10000000) + cache + } + + @Singleton + @Provides + @Named(IsColdStartPostInMemCache) + def providesIsColdStartPostInMemCache( + ): InProcessCache[Long, Boolean] = { + val BaseTTL = 4 + val TTL = (BaseTTL + Random.nextInt(3)).minutes + val cache: InProcessCache[Long, Boolean] = + new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 500000) + cache + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/LimiterModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/LimiterModule.scala new file mode 100644 index 000000000..75210b3a3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/LimiterModule.scala @@ -0,0 +1,27 @@ +package com.twitter.home_mixer.module + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.thriftmux.MethodBuilder +import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient +import com.twitter.inject.Injector +import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule +import com.twitter.limiter.{thriftscala => t} +import com.twitter.util.Duration + +object LimiterModule + extends ThriftMethodBuilderClientModule[ + t.LimitService.ServicePerEndpoint, + t.LimitService.MethodPerEndpoint + ] + with MtlsClient { + + override val label: String = "limiter" + override val dest: String = "/s/limiter/limiter" + + override protected def configureMethodBuilder( + injector: Injector, + methodBuilder: MethodBuilder + ): MethodBuilder = methodBuilder.withTimeoutPerRequest(200.milliseconds) + + override protected def sessionAcquisitionTimeout: Duration = 500.milliseconds +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala index fc0e282af..1d525d070 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala @@ -3,18 +3,26 @@ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanEndpoint +import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphManhattanEndpoint +import com.twitter.home_mixer.store.RTAMHStore import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.storage.client.manhattan.kv._ import com.twitter.timelines.config.ConfigUtils import com.twitter.util.Duration +import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} +import com.twitter.ml.api.DataRecord +import com.twitter.storehaus.ReadableStore + import javax.inject.Named import javax.inject.Singleton object ManhattanClientsModule extends TwitterModule with ConfigUtils { private val ApolloDest = "/s/manhattan/apollo.native-thrift" + private val BalterDest = "/s/manhattan/baltar.native-thrift" private final val Timeout = "mh_real_graph.timeout" flag[Duration](Timeout, 150.millis, "Timeout total") @@ -38,4 +46,33 @@ object ManhattanClientsModule extends TwitterModule with ConfigUtils { .defaultMaxTimeout(timeout) .build() } + + @Provides + @Singleton + @Named(RTAManhattanEndpoint) + def providesRTAManhattanEndpoint( + @Flag(Timeout) timeout: Duration, + serviceIdentifier: ServiceIdentifier + ): ManhattanKVEndpoint = { + lazy val client = ManhattanKVClient( + appId = "timelines_real_time_aggregates", + dest = BalterDest, + mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier = serviceIdentifier), + label = "rta-test" + ) + + ManhattanKVEndpointBuilder(client) + .maxRetryCount(2) + .defaultMaxTimeout(timeout) + .build() + } + + @Provides + @Singleton + @Named(RTAManhattanStore) + def providesRTAManhattanStore( + @Named(RTAManhattanEndpoint) manhattanKVEndpoint: ManhattanKVEndpoint + ): Option[ReadableStore[thrift.AggregationKey, DataRecord]] = { + Some(new RTAMHStore(manhattanKVEndpoint)) + } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala index 5668ba0ee..2974a5e49 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala @@ -1,5 +1,6 @@ package com.twitter.home_mixer.module +import com.google.common.primitives.Longs import com.google.inject.Provides import com.twitter.bijection.Injection import com.twitter.bijection.scrooge.BinaryScalaCodec @@ -16,7 +17,6 @@ import com.twitter.manhattan.v1.{thriftscala => mh} import com.twitter.ml.api.{thriftscala => ml} import com.twitter.ml.featurestore.lib.UserId import com.twitter.ml.featurestore.{thriftscala => fs} -import com.twitter.onboarding.relevance.features.{thriftjava => rf} import com.twitter.product_mixer.shared_library.manhattan_client.ManhattanClientBuilder import com.twitter.scalding_internal.multiformat.format.keyval.KeyValInjection.ScalaBinaryThrift import com.twitter.search.common.constants.{thriftscala => scc} @@ -33,8 +33,8 @@ import com.twitter.storage.client.manhattan.bijections.Bijections import com.twitter.storehaus_internal.manhattan.ManhattanClusters import com.twitter.timelines.author_features.v1.{thriftjava => af} import com.twitter.timelines.suggests.common.dense_data_record.{thriftscala => ddr} -import com.twitter.user_session_store.{thriftscala => uss_scala} import com.twitter.user_session_store.{thriftjava => uss} +import com.twitter.user_session_store.{thriftscala => uss_scala} import com.twitter.util.Duration import com.twitter.util.Try import java.nio.ByteBuffer @@ -57,11 +57,33 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { override def from(b: ByteBuffer): Try[Long] = ??? } + private val LongUserIdKeyTransformer = new Transformer[Long, ByteBuffer] { + override def to(userId: Long): Try[ByteBuffer] = { + Try(ByteBuffer.wrap(Longs.toByteArray(userId))) + } + override def from(b: ByteBuffer): Try[Long] = ??? + } + private val FloatTensorTransformer = new Transformer[ByteBuffer, ml.FloatTensor] { override def to(input: ByteBuffer): Try[ml.FloatTensor] = { val floatTensor = TensorFlowUtil.embeddingByteBufferToFloatTensor(input) Try(floatTensor) } + override def from(b: ml.FloatTensor): Try[ByteBuffer] = ??? + } + + private val EmbeddingTransformer = new Transformer[ByteBuffer, ml.FloatTensor] { + override def to(input: ByteBuffer): Try[ml.FloatTensor] = { + Try.fromScala( + Bijections + .BinaryScalaInjection(ml.Embedding).andThen(Bijections.byteBuffer2Buf.inverse).invert( + input).map { embedding => + embedding.tensor.map { tensor => + TensorFlowUtil.embeddingNoHeaderByteBufferToFloatTensor(tensor.content) + }.get + } + ) + } override def from(b: ml.FloatTensor): Try[ByteBuffer] = ??? } @@ -137,29 +159,6 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { } // non-cached manhattan repositories - - @Provides - @Singleton - @Named(MetricCenterUserCountingFeatureRepository) - def providesMetricCenterUserCountingFeatureRepository( - @Named(ManhattanStarbuckClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, rf.MCUserCountingFeatures] = { - - val valueTransformer = ThriftCodec - .toBinary[rf.MCUserCountingFeatures] - .toByteBufferTransformer() - .flip - - batchedManhattanKeyValueRepository[Long, rf.MCUserCountingFeatures]( - client = client, - keyTransformer = LongKeyTransformer, - valueTransformer = valueTransformer, - appId = "wtf_ml", - dataset = "mc_user_counting_features_v0_starbuck", - timeoutInMillis = 100 - ) - } - /** * A repository of the offline aggregate feature metadata necessary to decode * DenseCompactDataRecords. @@ -218,7 +217,7 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { @Singleton @Named(RealGraphFeatureRepository) def providesRealGraphFeatureRepository( - @Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint + @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint ): Repository[Long, Option[uss_scala.UserSession]] = { val valueTransformer = CompactScalaCodec(uss_scala.UserSession).toByteBufferTransformer().flip @@ -228,7 +227,7 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { keyTransformer = LongKeyTransformer, valueTransformer = valueTransformer, appId = "real_graph", - dataset = "split_real_graph_features", + dataset = "real_graph_user_features", timeoutInMillis = 100, ) ) @@ -261,14 +260,10 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { cacheClient = cacheClient, cachePrefix = "AuthorFeatureHydrator", ttl = 12.hours, - valueInjection = valueInjection) - - buildInProcessCachedRepository( - keyValueRepository = remoteCacheRepo, - ttl = 15.minutes, - size = 8000, valueInjection = valueInjection ) + + remoteCacheRepo } @Provides @@ -373,6 +368,23 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { ) } + @Provides + @Singleton + @Named(TwhinRebuildUserEngagementFeatureRepository) + def providesTwhinRebuildUserEngagementFeatureRepository( + @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint + ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { + + batchedManhattanKeyValueRepository( + client = client, + keyTransformer = LongUserIdKeyTransformer, + valueTransformer = EmbeddingTransformer, + appId = "twhin_embeddings_apollo", + dataset = "twhin_refreshed_user_eng_emb", + timeoutInMillis = 100 + ) + } + private def buildMemCachedRepository[K, V]( keyValueRepository: KeyValueRepository[Seq[K], K, V], cacheClient: Memcache, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MediaClusterId88Module.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MediaClusterId88Module.scala new file mode 100644 index 000000000..045d1256d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MediaClusterId88Module.scala @@ -0,0 +1,21 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClusterId88Store +import com.twitter.home_mixer.store.MediaClusterId88Store +import com.twitter.inject.TwitterModule +import com.twitter.storehaus.ReadableStore +import javax.inject.Named +import javax.inject.Singleton + +object MediaClusterId88Module extends TwitterModule { + + @Provides + @Singleton + @Named(MediaClusterId88Store) + def providesMediaClusterId88Store( + mediaClusterId88Store: MediaClusterId88Store + ): ReadableStore[Long, Long] = { + mediaClusterId88Store.clusterIdStore + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MediaClusterIdModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MediaClusterIdModule.scala new file mode 100644 index 000000000..1627d12cb --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MediaClusterIdModule.scala @@ -0,0 +1,21 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClusterId95Store +import com.twitter.home_mixer.store.MediaClusterId95Store +import com.twitter.inject.TwitterModule +import com.twitter.storehaus.ReadableStore +import javax.inject.Named +import javax.inject.Singleton + +object MediaClusterId95Module extends TwitterModule { + + @Provides + @Singleton + @Named(MediaClusterId95Store) + def providesMediaClusterId95Store( + mediaClusterId95Store: MediaClusterId95Store + ): ReadableStore[Long, Long] = { + mediaClusterId95Store.clusterIdStore + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala index 8afafbfb7..fb04ade59 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala @@ -5,14 +5,24 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Memcached import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.EntityRealGraphClientStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.HomeAuthorFeaturesCacheClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelinesRealTimeAggregateClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureCacheClient import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder +import com.twitter.servo.cache.ExpiringLruInProcessCache +import com.twitter.servo.cache.FinagleMemcache import com.twitter.servo.cache.FinagleMemcacheFactory +import com.twitter.servo.cache.HotKeyMemcacheClient import com.twitter.servo.cache.Memcache +import com.twitter.storehaus.ReadableStore +import com.twitter.wtf.entity_real_graph.summingbird.client.EntityRealGraphClient +import com.twitter.wtf.entity_real_graph.summingbird.common.config.Configs.Environment +import com.twitter.wtf.entity_real_graph.{thriftscala => erg} +import com.twitter.finagle.memcached.compressing.scheme.Lz4 +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TvRealTimeAggregateClient import javax.inject.Named import javax.inject.Singleton @@ -30,7 +40,8 @@ object MemcachedFeatureRepositoryModule extends TwitterModule { serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Memcache = { - val rawClient = MemcachedClientBuilder.buildRawMemcachedClient( + val cacheClient = MemcachedClientBuilder.buildMemcachedClient( + destName = "/s/cache/timelines_real_time_aggregates:twemcaches", numTries = 3, numConnections = 1, requestTimeout = 100.milliseconds, @@ -41,7 +52,13 @@ object MemcachedFeatureRepositoryModule extends TwitterModule { statsReceiver = statsReceiver ) - buildMemcacheClient(rawClient, "/s/cache/timelines_real_time_aggregates:twemcaches") + val hotkeyCacheClient = new HotKeyMemcacheClient( + proxyClient = cacheClient, + inProcessCache = new ExpiringLruInProcessCache(ttl = 15.minute, maximumSize = 75000), + statsReceiver = statsReceiver.scope(TimelinesRealTimeAggregateClient).scope("inProcess") + ) + + new FinagleMemcache(hotkeyCacheClient, memcacheKeyHasher) } @Provides @@ -59,7 +76,8 @@ object MemcachedFeatureRepositoryModule extends TwitterModule { connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver + statsReceiver = statsReceiver, + compressionScheme = Lz4 ) buildMemcacheClient(cacheClient, "/s/cache/timelines_author_features:twemcaches") @@ -107,6 +125,35 @@ object MemcachedFeatureRepositoryModule extends TwitterModule { buildMemcacheClient(cacheClient, "/s/cache/realtime_interactive_graph_prod_v2:twemcaches") } + @Provides + @Singleton + @Named(TvRealTimeAggregateClient) + def providesTvRealTimeAggregateClient( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): Memcache = { + val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( + numTries = 1, + numConnections = 1, + requestTimeout = 200.milliseconds, + globalTimeout = 200.milliseconds, + connectTimeout = 200.milliseconds, + acquisitionTimeout = 200.milliseconds, + serviceIdentifier = serviceIdentifier, + statsReceiver = statsReceiver + ) + buildMemcacheClient(cacheClient, "/srv#/prod/local/cache/tv_real_time_aggregates:twemcaches") + } + + @Provides + @Singleton + @Named(EntityRealGraphClientStore) + def providesEntityRealGraphClient( + serviceIdentifier: ServiceIdentifier + ): ReadableStore[erg.EntityRealGraphRequest, erg.EntityRealGraphResponse] = { + EntityRealGraphClient(Environment.withName("prod"), serviceIdentifier, Some(250.millis)) + } + private def buildMemcacheClient(cacheClient: Memcached.Client, dest: String): Memcache = FinagleMemcacheFactory( client = cacheClient, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedScoredCandidateFeaturesStoreModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedScoredCandidateFeaturesStoreModule.scala new file mode 100644 index 000000000..262ecb85e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedScoredCandidateFeaturesStoreModule.scala @@ -0,0 +1,84 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.inject.TwitterModule +import com.twitter.storehaus.Store +import com.twitter.timelines.clients.memcache_common.StorehausMemcacheConfig +import com.twitter.timelines.ml.cont_train.common.client.scored_candidate_features_cache.ScoredCandidateFeaturesMemcacheBuilder +import com.twitter.timelines.served_candidates_logging.{thriftscala => scl} +import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pdr} +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MemcacheCandidateFeaturesStore +import com.twitter.home_mixer.param.HomeMixerInjectionNames.MemcacheVideoCandidateFeaturesStore +import com.twitter.finagle.memcached.compressing.scheme.Lz4 +import javax.inject.Named +import javax.inject.Singleton + +object MemcachedScoredCandidateFeaturesStoreModule extends TwitterModule { + + private val ScoredTweetsProdDestName = + "/srv#/prod/local/cache/timelines_scored_candidate_features:twemcaches" + private val ScoredTweetsStagingDestName = + "/srv#/test/local/cache/twemcache_timelines_scored_candidate_features:twemcaches" + + private val ScoredVideoProdDestName = + "/srv#/prod/local/cache/timelines_scored_video_candidate_features:twemcaches" + private val ScoredVideoStagingDestName = + "/srv#/test/local/cache/twemcache_timelines_scored_video_candidate_features:twemcaches" + + @Singleton + @Provides + @Named(MemcacheCandidateFeaturesStore) + def providesMemcachedScoredCandidateFeaturesStore( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): Store[scl.CandidateFeatureKey, pdr.PolyDataRecord] = { + val destName = serviceIdentifier.environment.toLowerCase match { + case "prod" => ScoredTweetsProdDestName + case _ => ScoredTweetsStagingDestName + } + buildCacheClient(serviceIdentifier, statsReceiver, destName) + } + + @Singleton + @Provides + @Named(MemcacheVideoCandidateFeaturesStore) + def providesMemcachedScoredVideoCandidateFeaturesStore( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): Store[scl.CandidateFeatureKey, pdr.PolyDataRecord] = { + val destName = serviceIdentifier.environment.toLowerCase match { + case "prod" => ScoredVideoProdDestName + case _ => ScoredVideoStagingDestName + } + buildCacheClient(serviceIdentifier, statsReceiver, destName) + } + + private def buildCacheClient( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver, + destName: String, + ): Store[scl.CandidateFeatureKey, pdr.PolyDataRecord] = { + + new ScoredCandidateFeaturesMemcacheBuilder( + config = StorehausMemcacheConfig( + destName = destName, + keyPrefix = "", + requestTimeout = 200.milliseconds, + numTries = 2, + globalTimeout = 500.milliseconds, + tcpConnectTimeout = 20.milliseconds, + connectionAcquisitionTimeout = 150.milliseconds, + numPendingRequests = 200, + isReadOnly = false, + serviceIdentifier = serviceIdentifier, + numConnections = 1, + compressionScheme = Lz4 + ), + ttl = 5.minute, + statsReceiver = statsReceiver + ).build() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala index 60d580a73..5fb4b5857 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala @@ -6,27 +6,76 @@ import com.twitter.finagle.Http import com.twitter.finagle.grpc.FinagleChannelBuilder import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsStackClientSyntax +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecap +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapGPU +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapRealtime +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapSecondary +import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapVideo import com.twitter.inject.TwitterModule import com.twitter.timelines.clients.predictionservice.PredictionGRPCService import com.twitter.util.Duration import io.grpc.ManagedChannel +import javax.inject.Named import javax.inject.Singleton object NaviModelClientModule extends TwitterModule { + private val Authority = "rustserving" + private val MaxRetryAttempts = 2 + private val MaxPredictionTimeoutMs: Duration = 700.millis + private val ConnectTimeoutMs: Duration = 200.millis + private val AcquisitionTimeoutMs: Duration = 500.millis + + @Singleton + @Named(NaviModelClientHomeRecap) + @Provides + def providesHomeRecapPredictionGRPCService( + serviceIdentifier: ServiceIdentifier, + ): PredictionGRPCService = { + providesPredictionGRPCService(serviceIdentifier, "navi_home_recap_onnx") + } + @Singleton + @Named(NaviModelClientHomeRecapSecondary) @Provides - def providesPredictionGRPCService( + def providesHomeRecapSecondaryPredictionGRPCService( serviceIdentifier: ServiceIdentifier, ): PredictionGRPCService = { - // Wily path to the ML Model service (e.g. /s/ml-serving/navi-explore-ranker). - val modelPath = "/s/ml-serving/navi_home_recap_onnx" + providesPredictionGRPCService(serviceIdentifier, "navi_home_recap_onnx_2") + } - val MaxPredictionTimeoutMs: Duration = 500.millis - val ConnectTimeoutMs: Duration = 200.millis - val AcquisitionTimeoutMs: Duration = 500.millis - val MaxRetryAttempts: Int = 2 + @Singleton + @Named(NaviModelClientHomeRecapRealtime) + @Provides + def providesHomeRecapRealtimePredictionGRPCService( + serviceIdentifier: ServiceIdentifier, + ): PredictionGRPCService = { + providesPredictionGRPCService(serviceIdentifier, "navi_home_realtime_recap_onnx") + } + @Singleton + @Named(NaviModelClientHomeRecapGPU) + @Provides + def providesHomeRecapGPUPredictionGRPCService( + serviceIdentifier: ServiceIdentifier, + ): PredictionGRPCService = { + providesPredictionGRPCService(serviceIdentifier, "navi_home_recap_onnx_v100") + } + + @Singleton + @Named(NaviModelClientHomeRecapVideo) + @Provides + def providesHomeRecapVideoPredictionGRPCService( + serviceIdentifier: ServiceIdentifier, + ): PredictionGRPCService = { + providesPredictionGRPCService(serviceIdentifier, "navi_home_recap_video_onnx") + } + + private def providesPredictionGRPCService( + serviceIdentifier: ServiceIdentifier, + naviClusterName: String + ): PredictionGRPCService = { + val modelPath = s"/s/ml-serving/$naviClusterName" val client = Http.client .withLabel(modelPath) .withMutualTls(serviceIdentifier) @@ -37,7 +86,7 @@ object NaviModelClientModule extends TwitterModule { val channel: ManagedChannel = FinagleChannelBuilder .forTarget(modelPath) - .overrideAuthority("rustserving") + .overrideAuthority(Authority) .maxRetryAttempts(MaxRetryAttempts) .enableRetryForStatus(io.grpc.Status.RESOURCE_EXHAUSTED) .enableRetryForStatus(io.grpc.Status.UNKNOWN) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala index b9e315acc..9f88945fb 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala @@ -7,17 +7,30 @@ import com.twitter.finagle.service.Retries import com.twitter.finagle.service.RetryPolicy import com.twitter.finagle.ssl.OpportunisticTls import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithDefaultTimeout +import com.twitter.home_mixer.param.HomeMixerInjectionNames.StratoClientWithLongTimeout +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TesBatchedStratoClient import com.twitter.inject.TwitterModule import com.twitter.strato.client.Client import com.twitter.strato.client.Strato +import com.twitter.strato.rpc.ClientBucketingStrategy import com.twitter.util.Try import javax.inject.Named import javax.inject.Singleton +/** + * Strato Finagle config is copied from TLX + */ object OptimizedStratoClientModule extends TwitterModule { - private val ModerateStratoServerClientRequestTimeout = 500.millis + private val StratoClientConnectionTimeout = 200.millis + private val StratoClientAcquisitionTimeout = 500.millis + private val DefaultStratoClientRequestTimeout = 280.millis + private val StratoClientLongRequestTimeout = 1000.millis + private val ModerateStratoClientRequestTimeout = 600.millis + private val LongStratoClientRequestTimeout = 1000.millis private val DefaultRetryPartialFunction: PartialFunction[Try[Nothing], Boolean] = RetryPolicy.TimeoutAndWriteExceptionsOnly @@ -26,21 +39,97 @@ object OptimizedStratoClientModule extends TwitterModule { protected def mkRetryPolicy(tries: Int): RetryPolicy[Try[Nothing]] = RetryPolicy.tries(tries, DefaultRetryPartialFunction) + @Singleton + @Provides + @Named(BatchedStratoClientWithDefaultTimeout) + def providesDefaultStratoClient( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): Client = { + Strato.client + .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) + .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) + .withTransport.connectTimeout(StratoClientConnectionTimeout) + .withRequestTimeout(DefaultStratoClientRequestTimeout) + .withPerRequestTimeout(DefaultStratoClientRequestTimeout) + .configured(Retries.Policy(mkRetryPolicy(1))) + .withStatsReceiver(statsReceiver.scope("default_strato_client")) + .build() + } + + @Singleton + @Provides + @Named(StratoClientWithLongTimeout) + def providesLongStratoClient( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): Client = { + Strato.client + .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) + .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) + .withTransport.connectTimeout(StratoClientConnectionTimeout) + .withRequestTimeout(StratoClientLongRequestTimeout) + .withPerRequestTimeout(StratoClientLongRequestTimeout) + .configured(Retries.Policy(mkRetryPolicy(10))) + .withStatsReceiver(statsReceiver.scope("long_strato_client")) + .build() + } + @Singleton @Provides @Named(BatchedStratoClientWithModerateTimeout) - def providesStratoClient( + def providesModerateTimeoutStratoClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Client = { Strato.client .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) - .withSession.acquisitionTimeout(500.milliseconds) - .withRequestTimeout(ModerateStratoServerClientRequestTimeout) - .withPerRequestTimeout(ModerateStratoServerClientRequestTimeout) + .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) + .withTransport.connectTimeout(StratoClientConnectionTimeout) + .withRequestTimeout(ModerateStratoClientRequestTimeout) + .withPerRequestTimeout(ModerateStratoClientRequestTimeout) + .withRpcBatchSize(64) + .configured(Retries.Policy(mkRetryPolicy(1))) + .withStatsReceiver(statsReceiver.scope("moderate_timeout_strato_client")) + .build() + } + + @Singleton + @Provides + @Named(BatchedStratoClientWithLongTimeout) + def providesLongTimeoutStratoClient( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): Client = { + Strato.client + .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) + .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) + .withTransport.connectTimeout(StratoClientConnectionTimeout) + .withRequestTimeout(LongStratoClientRequestTimeout) + .withPerRequestTimeout(LongStratoClientRequestTimeout) .withRpcBatchSize(5) .configured(Retries.Policy(mkRetryPolicy(1))) - .withStatsReceiver(statsReceiver.scope("strato_client")) + .withStatsReceiver(statsReceiver.scope("long_timeout_strato_client")) + .build() + } + + @Singleton + @Provides + @Named(TesBatchedStratoClient) + def providesTesStratoClient( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): Client = { + Strato.client + .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) + .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) + .withTransport.connectTimeout(StratoClientConnectionTimeout) + .withRequestTimeout(ModerateStratoClientRequestTimeout) + .withPerRequestTimeout(ModerateStratoClientRequestTimeout) + .withRpcBatchSize(140) + .withBucketingStrategy(ClientBucketingStrategy.ByArg) + .configured(Retries.Policy(mkRetryPolicy(1))) + .withStatsReceiver(statsReceiver.scope("tes_strato_client")) .build() } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PhoenixClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PhoenixClientModule.scala new file mode 100644 index 000000000..be085bb4e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PhoenixClientModule.scala @@ -0,0 +1,61 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.google.inject.name.Named +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment1 +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment2 +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment3 +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment4 +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment5 +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment6 +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment7 +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment8 +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Prod +import com.twitter.inject.TwitterModule +import io.grpc.ManagedChannel +import io.grpc.netty.NettyChannelBuilder +import java.util.concurrent.TimeUnit +import javax.inject.Singleton + +object PhoenixClientModule extends TwitterModule { + + val ChannelsPerHost = 10 + val XAIGrpcPort = 80 + val ProdEndpoint = "" + + private def buildChannels(endpoint: String): Seq[ManagedChannel] = { + val endpoints = Seq.fill(ChannelsPerHost)(endpoint) + endpoints.map { host => + NettyChannelBuilder + .forAddress(host, XAIGrpcPort) + .usePlaintext() + .keepAliveTime(60, TimeUnit.SECONDS) + .keepAliveTimeout(20, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true) + .initialFlowControlWindow(128 * 1024 * 1024) + .flowControlWindow(1024 * 1024 * 128) + .maxInboundMessageSize(20 * 1024 * 1024) + .build() + } + } + + @Provides + @Singleton + @Named("PhoenixClient") + private def getStub(): Map[PhoenixCluster.Value, Seq[ManagedChannel]] = { + val endpointMap = Map( + Prod -> ProdEndpoint, + Experiment1 -> "", + Experiment2 -> "", + Experiment3 -> "", + Experiment4 -> "", + Experiment5 -> "", + Experiment6 -> "", + Experiment7 -> "", + Experiment8 -> "", + ).withDefaultValue(ProdEndpoint) + + endpointMap.mapValues(buildChannels) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.scala index 7dc6a072d..668d76266 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.scala @@ -2,7 +2,7 @@ package com.twitter.home_mixer.module import com.google.inject.Provides import com.google.inject.name.Named -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphInNetworkScores +import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphInNetworkScoresOnPrem import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphManhattanEndpoint import com.twitter.home_mixer.store.RealGraphInNetworkScoresStore import com.twitter.inject.TwitterModule @@ -10,14 +10,13 @@ import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint import com.twitter.storehaus.ReadableStore import com.twitter.timelines.util.CommonTypes.ViewerId import com.twitter.wtf.candidate.thriftscala.Candidate - import javax.inject.Singleton object RealGraphInNetworkScoresModule extends TwitterModule { @Provides @Singleton - @Named(RealGraphInNetworkScores) + @Named(RealGraphInNetworkScoresOnPrem) def providesRealGraphInNetworkScoresFeaturesStore( @Named(RealGraphManhattanEndpoint) realGraphInNetworkScoresManhattanKVEndpoint: ManhattanKVEndpoint ): ReadableStore[ViewerId, Seq[Candidate]] = { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala index c3c545819..d49292933 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala @@ -11,6 +11,8 @@ import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionG import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelinesRealTimeAggregateClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicCountryEngagementCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicEngagementCache +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TvRealTimeAggregateClient +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TvVideoByUserTweetCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetCountryEngagementCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetEngagementCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwitterListEngagementCache @@ -32,20 +34,12 @@ import com.twitter.summingbird_internal.bijection.BatchPairImplicits import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKey import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKeyInjection import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig} - import javax.inject.Singleton object RealtimeAggregateFeatureRepositoryModule extends TwitterModule with RealtimeAggregateHelpers { - private val authorIdFeature = new Feature.Discrete("entities.source_author_id").getFeatureId - private val countryCodeFeature = new Feature.Text("geo.user_location.country_code").getFeatureId - private val listIdFeature = new Feature.Discrete("list.id").getFeatureId - private val userIdFeature = new Feature.Discrete("meta.user_id").getFeatureId - private val topicIdFeature = new Feature.Discrete("entities.topic_id").getFeatureId - private val tweetIdFeature = new Feature.Discrete("entities.source_tweet_id").getFeatureId - @Provides @Singleton @Named(UserTopicEngagementForNewUserCache) @@ -188,10 +182,30 @@ object RealtimeAggregateFeatureRepositoryModule underlyingKey ) } + + @Provides + @Singleton + @Named(TvVideoByUserTweetCache) + def providesTvVideoImpressionByUserTweetCache( + @Named(TvRealTimeAggregateClient) client: Memcache + ): ReadCache[(Long, Long), ml.DataRecord] = { + new KeyValueTransformingReadCache( + client, + dataRecordValueTransformer, + keyTransformD2(userIdFeature, tweetIdFeature) + ) + } } trait RealtimeAggregateHelpers { + val authorIdFeature = new Feature.Discrete("entities.source_author_id").getFeatureId + val countryCodeFeature = new Feature.Text("geo.user_location.country_code").getFeatureId + val listIdFeature = new Feature.Discrete("list.id").getFeatureId + val userIdFeature = new Feature.Discrete("meta.user_id").getFeatureId + val topicIdFeature = new Feature.Discrete("entities.topic_id").getFeatureId + val tweetIdFeature = new Feature.Discrete("entities.source_tweet_id").getFeatureId + private def customKeyBuilder[K](prefix: String, f: K => Array[Byte]): K => String = { // intentionally not implementing injection inverse because it is never used def g(arr: Array[Byte]) = ??? @@ -208,24 +222,38 @@ trait RealtimeAggregateHelpers { .compose((k: AggregationKey) => (k, defaultBatchID)) } - protected def keyTransformD1(f1: Long)(key: Long): String = { + def keyTransformD1AggregationKey(f1: Long)(key: Long): AggregationKey = { + AggregationKey(Map(f1 -> key), Map.empty) + } + + def keyTransformD1(f1: Long)(key: Long): String = { val aggregationKey = AggregationKey(Map(f1 -> key), Map.empty) keyEncoder(aggregationKey) } - protected def keyTransformD2(f1: Long, f2: Long)(keys: (Long, Long)): String = { + def keyTransformD2(f1: Long, f2: Long)(keys: (Long, Long)): String = { val (k1, k2) = keys val aggregationKey = AggregationKey(Map(f1 -> k1, f2 -> k2), Map.empty) keyEncoder(aggregationKey) } - protected def keyTransformD1T1(f1: Long, f2: Long)(keys: (Long, String)): String = { + def keyTransformD2AggregationKey(f1: Long, f2: Long)(keys: (Long, Long)): AggregationKey = { + val (k1, k2) = keys + AggregationKey(Map(f1 -> k1, f2 -> k2), Map.empty) + } + + def keyTransformD1T1(f1: Long, f2: Long)(keys: (Long, String)): String = { val (k1, k2) = keys val aggregationKey = AggregationKey(Map(f1 -> k1), Map(f2 -> k2)) keyEncoder(aggregationKey) } - protected val dataRecordValueTransformer: Transformer[DataRecord, Array[Byte]] = ThriftCodec + def keyTransformD1T1AggregationKey(f1: Long, f2: Long)(keys: (Long, String)): AggregationKey = { + val (k1, k2) = keys + AggregationKey(Map(f1 -> k1), Map(f2 -> k2)) + } + + val dataRecordValueTransformer: Transformer[DataRecord, Array[Byte]] = ThriftCodec .toCompact[ml.DataRecord] .toByteArrayTransformer() } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala index 4bed31c5c..f390a309c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala @@ -4,6 +4,7 @@ import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.ScoredTweetsCache import com.twitter.home_mixer.{thriftscala => t} import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder @@ -15,7 +16,9 @@ import com.twitter.servo.cache.ThriftSerializer import com.twitter.servo.cache.TtlCache import com.twitter.timelines.model.UserId import org.apache.thrift.protocol.TCompactProtocol +import com.twitter.finagle.memcached.compressing.scheme.Lz4 +import javax.inject.Named import javax.inject.Singleton object ScoredTweetsMemcacheModule extends TwitterModule { @@ -30,6 +33,7 @@ object ScoredTweetsMemcacheModule extends TwitterModule { private val userIdKeyTransformer: KeyTransformer[UserId] = (userId: UserId) => userId.toString @Singleton + @Named(ScoredTweetsCache) @Provides def providesScoredTweetsCache( serviceIdentifier: ServiceIdentifier, @@ -48,7 +52,8 @@ object ScoredTweetsMemcacheModule extends TwitterModule { connectTimeout = 100.milliseconds, acquisitionTimeout = 100.milliseconds, serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver.scope(ScopeName) + statsReceiver = statsReceiver.scope(ScopeName), + compressionScheme = Lz4 ) val underlyingCache = new FinagleMemcache(client) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredVideoTweetsMemcacheModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredVideoTweetsMemcacheModule.scala new file mode 100644 index 000000000..9e59f853f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredVideoTweetsMemcacheModule.scala @@ -0,0 +1,58 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.memcached.compressing.scheme.Lz4 +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.ScoredVideoTweetsCache +import com.twitter.home_mixer.{thriftscala => t} +import com.twitter.inject.TwitterModule +import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder +import com.twitter.servo.cache._ +import com.twitter.timelines.model.UserId +import org.apache.thrift.protocol.TCompactProtocol + +import javax.inject.Named +import javax.inject.Singleton + +object ScoredVideoTweetsMemcacheModule extends TwitterModule { + + private val ScopeName = "ScoredVideoTweetsCache" + private val destName = "/s/cache/explore_served_tweet_impressions:twemcaches" + private val scoredTweetsSerializer: Serializer[t.ScoredTweetsResponse] = + new ThriftSerializer[t.ScoredTweetsResponse]( + t.ScoredTweetsResponse, + new TCompactProtocol.Factory()) + + private val userIdKeyTransformer: KeyTransformer[UserId] = (userId: UserId) => + userId.toString + ":home-mixer" + + @Singleton + @Named(ScoredVideoTweetsCache) + @Provides + def providesScoredTweetsCache( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): TtlCache[UserId, t.ScoredTweetsResponse] = { + val client = MemcachedClientBuilder.buildMemcachedClient( + destName = destName, + numTries = 2, + numConnections = 1, + requestTimeout = 200.milliseconds, + globalTimeout = 400.milliseconds, + connectTimeout = 100.milliseconds, + acquisitionTimeout = 100.milliseconds, + serviceIdentifier = serviceIdentifier, + statsReceiver = statsReceiver.scope(ScopeName), + compressionScheme = Lz4 + ) + val underlyingCache = new FinagleMemcache(client) + + new KeyValueTransformingTtlCache( + underlyingCache = underlyingCache, + transformer = scoredTweetsSerializer, + underlyingKey = userIdKeyTransformer + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala index 99bf61630..b06b85f9f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala @@ -2,18 +2,27 @@ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.home_mixer.param.HomeMixerInjectionNames.CandidateFeaturesScribeEventPublisher +import com.twitter.finatra.kafka.interceptors.InstanceMetadataProducerInterceptor +import com.twitter.finatra.kafka.interceptors.PublishTimeProducerInterceptor +import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder +import com.twitter.finatra.kafka.serde.ScalaSerdes +import com.twitter.finatra.kafka.serde.UnKeyedSerde import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeEventPublisher -import com.twitter.home_mixer.param.HomeMixerInjectionNames.MinimumFeaturesScribeEventPublisher +import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeVideoEventPublisher import com.twitter.inject.TwitterModule import com.twitter.logpipeline.client.EventPublisherManager import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.logpipeline.client.serializers.EventLogMsgTBinarySerializer -import com.twitter.logpipeline.client.serializers.EventLogMsgThriftStructSerializer import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} import com.twitter.timelines.timeline_logging.{thriftscala => tl} +import com.twitter.util.Duration +import com.twitter.util.StorageUnit import javax.inject.Named import javax.inject.Singleton +import org.apache.kafka.clients.CommonClientConfigs +import org.apache.kafka.common.config.SaslConfigs +import org.apache.kafka.common.config.SslConfigs +import org.apache.kafka.common.security.auth.SecurityProtocol object ScribeEventPublisherModule extends TwitterModule { @@ -21,14 +30,28 @@ object ScribeEventPublisherModule extends TwitterModule { val ServedCandidatesLogCategory = "home_timeline_served_candidates_flattened" val ScoredCandidatesLogCategory = "home_timeline_scored_candidates" val ServedCommonFeaturesLogCategory = "tq_served_common_features_offline" - val ServedCandidateFeaturesLogCategory = "tq_served_candidate_features_offline" - val ServedMinimumFeaturesLogCategory = "tq_served_minimum_features_offline" + val ServedVideoCommonFeaturesLogCategory = "tq_served_video_common_features_offline" @Provides @Singleton def providesClientEventsScribeEventPublisher: EventPublisher[ca.LogEvent] = { - val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[ca.LogEvent]() - EventPublisherManager.buildScribeLogPipelinePublisher(ClientEventLogCategory, serializer) + val builder = FinagleKafkaProducerBuilder() + .dest(s"/s/kafka/client-events:kafka-tls") + .keySerializer(UnKeyedSerde.serializer) + .valueSerializer(ScalaSerdes.Thrift[ca.LogEvent].serializer) + .clientId("home_mixer_client_event_publisher") + .linger(Duration.fromMilliseconds(16)) + .batchSize(StorageUnit.fromKilobytes(64)) + .deliveryTimeout(Duration.fromMilliseconds(30000)) + .requestTimeout(Duration.fromMilliseconds(25000)) + .withConfig(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.toString) + .interceptor[PublishTimeProducerInterceptor] + .interceptor[InstanceMetadataProducerInterceptor] + .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "") + .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) + .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") + .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") + EventPublisherManager.buildKafkaLogPipelinePublisher(builder, ClientEventLogCategory) } @Provides @@ -43,35 +66,55 @@ object ScribeEventPublisherModule extends TwitterModule { @Provides @Singleton - @Named(CandidateFeaturesScribeEventPublisher) - def providesCandidateFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { + @Named(CommonFeaturesScribeVideoEventPublisher) + def providesCommonFeaturesScribeVideoEventPublisher: EventPublisher[pldr.PolyDataRecord] = { val serializer = EventLogMsgTBinarySerializer.getNewSerializer EventPublisherManager.buildScribeLogPipelinePublisher( - ServedCandidateFeaturesLogCategory, - serializer) - } - - @Provides - @Singleton - @Named(MinimumFeaturesScribeEventPublisher) - def providesMinimumFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { - val serializer = EventLogMsgTBinarySerializer.getNewSerializer - EventPublisherManager.buildScribeLogPipelinePublisher( - ServedMinimumFeaturesLogCategory, + ServedVideoCommonFeaturesLogCategory, serializer) } @Provides @Singleton def providesServedCandidatesScribeEventPublisher: EventPublisher[tl.ServedEntry] = { - val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[tl.ServedEntry]() - EventPublisherManager.buildScribeLogPipelinePublisher(ServedCandidatesLogCategory, serializer) + val builder = FinagleKafkaProducerBuilder() + .dest("/s/kafka/timeline:kafka-tls") + .keySerializer(UnKeyedSerde.serializer) + .valueSerializer(ScalaSerdes.Thrift[tl.ServedEntry].serializer) + .clientId(s"$ServedCandidatesLogCategory-publisher") + .linger(Duration.fromMilliseconds(16)) + .batchSize(StorageUnit.fromKilobytes(64)) + .deliveryTimeout(Duration.fromMilliseconds(30000)) + .requestTimeout(Duration.fromMilliseconds(25000)) + .withConfig(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.toString) + .interceptor[PublishTimeProducerInterceptor] + .interceptor[InstanceMetadataProducerInterceptor] + .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "") + .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) + .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") + .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") + EventPublisherManager.buildKafkaLogPipelinePublisher(builder, ServedCandidatesLogCategory) } @Provides @Singleton def provideScoredCandidatesScribeEventPublisher: EventPublisher[tl.ScoredCandidate] = { - val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[tl.ScoredCandidate]() - EventPublisherManager.buildScribeLogPipelinePublisher(ScoredCandidatesLogCategory, serializer) + val builder = FinagleKafkaProducerBuilder() + .dest("/s/kafka/timeline:kafka-tls") + .keySerializer(UnKeyedSerde.serializer) + .valueSerializer(ScalaSerdes.Thrift[tl.ScoredCandidate].serializer) + .clientId(s"$ScoredCandidatesLogCategory-publisher") + .linger(Duration.fromMilliseconds(16)) + .batchSize(StorageUnit.fromKilobytes(64)) + .deliveryTimeout(Duration.fromMilliseconds(30000)) + .requestTimeout(Duration.fromMilliseconds(25000)) + .withConfig(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.toString) + .interceptor[PublishTimeProducerInterceptor] + .interceptor[InstanceMetadataProducerInterceptor] + .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "") + .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) + .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") + .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") + EventPublisherManager.buildKafkaLogPipelinePublisher(builder, ScoredCandidatesLogCategory) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala index dcbf62451..f1867a66e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala @@ -10,38 +10,25 @@ import com.twitter.graph_feature_service.{thriftscala => gfs} import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.InterestsThriftServiceClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieContentRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserFollowedTopicIdsRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil -import com.twitter.home_mixer.util.tweetypie.RequestFields import com.twitter.inject.TwitterModule import com.twitter.interests.{thriftscala => int} -import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder import com.twitter.product_mixer.shared_library.thrift_client.Idempotent import com.twitter.recos.recos_common.{thriftscala => rc} import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.servo.cache.Cached -import com.twitter.servo.cache.CachedSerializer -import com.twitter.servo.cache.FinagleMemcacheFactory -import com.twitter.servo.cache.MemcacheCacheFactory -import com.twitter.servo.cache.NonLockingCache -import com.twitter.servo.cache.ThriftSerializer +import com.twitter.servo.cache._ import com.twitter.servo.keyvalue.KeyValueResultBuilder -import com.twitter.servo.repository.CachingKeyValueRepository import com.twitter.servo.repository.ChunkingStrategy import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueResult -import com.twitter.servo.repository.keysAsQuery -import com.twitter.spam.rtf.{thriftscala => sp} -import com.twitter.tweetypie.{thriftscala => tp} import com.twitter.util.Future import com.twitter.util.Return import javax.inject.Named import javax.inject.Singleton -import org.apache.thrift.protocol.TCompactProtocol object ThriftFeatureRepositoryModule extends TwitterModule { @@ -148,82 +135,6 @@ object ThriftFeatureRepositoryModule extends TwitterModule { toRepositoryBatchWithView(lookup, chunkSize = 200) } - @Provides - @Singleton - @Named(TweetypieContentRepository) - def providesTweetypieContentRepository( - clientId: ClientId, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): KeyValueRepository[Seq[Long], Long, tp.Tweet] = { - val client = FinagleThriftClientBuilder - .buildFinagleMethodPerEndpoint[ - tp.TweetService.ServicePerEndpoint, - tp.TweetService.MethodPerEndpoint]( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = "/s/tweetypie/tweetypie", - label = "tweetypie-content-repo", - statsReceiver = statsReceiver, - idempotency = Idempotent(1.percent), - timeoutPerRequest = 300.milliseconds, - timeoutTotal = 500.milliseconds - ) - - def lookup(tweetIds: Seq[Long]): Future[Seq[Option[tp.Tweet]]] = { - val getTweetFieldsOptions = tp.GetTweetFieldsOptions( - tweetIncludes = RequestFields.ContentFields, - includeRetweetedTweet = false, - includeQuotedTweet = false, - forUserId = None, - safetyLevel = Some(sp.SafetyLevel.FilterNone), - visibilityPolicy = tp.TweetVisibilityPolicy.NoFiltering - ) - - val request = tp.GetTweetFieldsRequest(tweetIds = tweetIds, options = getTweetFieldsOptions) - - client.getTweetFields(request).map { results => - results.map { - case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), _, _) => - Some(found.tweet) - case _ => None - } - } - } - - val keyValueRepository = toRepositoryBatch(lookup, chunkSize = 20) - - val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 1, - numConnections = 1, - requestTimeout = 200.milliseconds, - globalTimeout = 200.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - val finagleMemcacheFactory = - FinagleMemcacheFactory(cacheClient, "/s/cache/home_content_features:twemcaches") - val cacheValueTransformer = - new ThriftSerializer[tp.Tweet](tp.Tweet, new TCompactProtocol.Factory()) - val cachedSerializer = CachedSerializer.binary(cacheValueTransformer) - - val cache = MemcacheCacheFactory( - memcache = finagleMemcacheFactory(), - ttl = 48.hours - )[Long, Cached[tp.Tweet]](cachedSerializer) - - val lockingCache = new NonLockingCache(cache) - val cachedKeyValueRepository = new CachingKeyValueRepository( - keyValueRepository, - lockingCache, - keysAsQuery[Long] - ) - cachedKeyValueRepository - } - @Provides @Singleton @Named(GraphTwoHopRepository) @@ -284,7 +195,7 @@ object ThriftFeatureRepositoryModule extends TwitterModule { tweetIds = Some(tweetIds), clientId = Some(clientId.name), authorScoreMap = None, - tensorflowModel = Some("timelines_rectweet_replica") + tensorflowModel = Some("timelines_unified_prod") ) client diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TvWatchHistoryCacheClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TvWatchHistoryCacheClientModule.scala new file mode 100644 index 000000000..937051056 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TvWatchHistoryCacheClientModule.scala @@ -0,0 +1,72 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hashing.KeyHasher +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TvWatchedVideoIdsKeyCacheStore +import com.twitter.inject.TwitterModule +import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder +import com.twitter.servo.cache.FinagleMemcache +import com.twitter.servo.cache.KeyValueTransformingTtlCache +import com.twitter.servo.cache.ObservableTtlCache +import com.twitter.servo.cache.TtlCache +import com.twitter.servo.util.Transformer +import com.twitter.util.Try +import java.nio.ByteBuffer +import javax.inject.Named +import javax.inject.Singleton +import scala.collection.mutable.ArrayBuffer + +object TvWatchHistoryCacheClientModule extends TwitterModule { + @Provides + @Singleton + @Named(TvWatchedVideoIdsKeyCacheStore) + def providesEntityRealGraphClient( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): TtlCache[Long, Seq[Long]] = { + val scopeName = "TvWatchedVideoIds" + val memCacheClient = MemcachedClientBuilder.buildMemcachedClient( + destName = "/srv#/prod/local/cache/related_videos:twemcaches", + numTries = 1, + numConnections = 1, + requestTimeout = 50.milliseconds, + globalTimeout = 100.milliseconds, + connectTimeout = 100.milliseconds, + acquisitionTimeout = 100.milliseconds, + serviceIdentifier = serviceIdentifier, + statsReceiver = statsReceiver, + keyHasher = Some(KeyHasher.FNV1A_64) + ) + + val longSeqTransformer: Transformer[Seq[Long], Array[Byte]] = + new Transformer[Seq[Long], Array[Byte]] { + def deserialize(input: Array[Byte]): ArrayBuffer[Long] = { + val buffer = ByteBuffer.wrap(input) + val resultBuffer = ArrayBuffer[Long]() + while (buffer.hasRemaining) { + val longValue = buffer.getLong() + resultBuffer += longValue + } + resultBuffer + } + + override def from(input: Array[Byte]): Try[Seq[Long]] = Try(deserialize(input)) + + override def to(b: Seq[Long]): Try[Array[Byte]] = ??? + } + val keyValueCache = new KeyValueTransformingTtlCache[Long, String, Seq[Long], Array[Byte]]( + underlyingCache = new FinagleMemcache(memCacheClient), + transformer = longSeqTransformer, + underlyingKey = { key: Long => key.toString } + ) + ObservableTtlCache( + underlyingCache = keyValueCache, + statsReceiver = statsReceiver.scope(scopeName), + windowSize = 1000, + name = scopeName + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetWatchTimeMetadataModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetWatchTimeMetadataModule.scala new file mode 100644 index 000000000..27377bcf2 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetWatchTimeMetadataModule.scala @@ -0,0 +1,23 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetWatchTimeMetadataStore +import com.twitter.home_mixer.store.TweetWatchTimeMetadataStore +import com.twitter.inject.TwitterModule +import com.twitter.storehaus.ReadableStore +import com.twitter.twistly.thriftscala.VideoViewEngagementType +import com.twitter.twistly.thriftscala.WatchTimeMetadata +import javax.inject.Named +import javax.inject.Singleton + +object TweetWatchTimeMetadataModule extends TwitterModule { + + @Provides + @Singleton + @Named(TweetWatchTimeMetadataStore) + def providesTweetWatchTimeMetadataStore( + tweetWatchTimeMetadataStore: TweetWatchTimeMetadataStore + ): ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata] = { + tweetWatchTimeMetadataStore.tweetWatchTimeMetadataStore + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala index 13283e4b3..fbc06163f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala @@ -1,20 +1,32 @@ package com.twitter.home_mixer.module -import com.google.inject.name.Named import com.google.inject.Provides +import com.google.inject.name.Named import com.twitter.conversions.DurationOps.RichDuration +import com.twitter.cproxy.{thriftscala => cp} +import com.twitter.deferredrpc.client.DeferredThriftService +import com.twitter.deferredrpc.thrift.Datacenter +import com.twitter.deferredrpc.thrift.DeferredRPC +import com.twitter.deferredrpc.thrift.Target +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.builder.ClientBuilder import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder +import com.twitter.servo.cache.ExpiringLruInProcessCache import com.twitter.servo.cache.FinagleMemcache +import com.twitter.servo.cache.HotKeyMemcacheClient import com.twitter.servo.cache.KeyTransformer import com.twitter.servo.cache.KeyValueTransformingTtlCache import com.twitter.servo.cache.ObservableTtlCache import com.twitter.servo.cache.Serializer import com.twitter.servo.cache.ThriftSerializer import com.twitter.servo.cache.TtlCache +import com.twitter.servo.util.Gate +import com.twitter.timelinemixer.clients.rankedtweetcaching.CproxyTtlCacheWrapper import com.twitter.tweetypie.{thriftscala => tp} import javax.inject.Singleton import org.apache.thrift.protocol.TCompactProtocol @@ -22,11 +34,13 @@ import org.apache.thrift.protocol.TCompactProtocol object TweetypieStaticEntitiesCacheClientModule extends TwitterModule { private val ScopeName = "TweetypieStaticEntitiesMemcache" - private val ProdDest = "/srv#/prod/local/cache/timelinescorer_tweet_core_data:twemcaches" + private val ProdDest = "/s/cache/timelinescorer_tweet_core_data:twemcaches" + private val ProdKrpcDest = "/s/kafka-shared/krpc-server-cache" + private val DevelKrpcDest = "/srv#/staging/local/kafka-shared/krpc-server-custdevel" - private val tweetsSerializer: Serializer[tp.Tweet] = { + private val tweetsSerializer: Serializer[tp.Tweet] = new ThriftSerializer[tp.Tweet](tp.Tweet, new TCompactProtocol.Factory()) - } + private val keyTransformer: KeyTransformer[Long] = { tweetId => tweetId.toString } @Provides @@ -40,31 +54,65 @@ object TweetypieStaticEntitiesCacheClientModule extends TwitterModule { destName = ProdDest, numTries = 1, numConnections = 1, - requestTimeout = 50.milliseconds, - globalTimeout = 100.milliseconds, - connectTimeout = 100.milliseconds, - acquisitionTimeout = 100.milliseconds, + requestTimeout = 200.milliseconds, + globalTimeout = 200.milliseconds, + connectTimeout = 200.milliseconds, + acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver ) - mkCache(new FinagleMemcache(memCacheClient), statsReceiver) - } - private def mkCache( - finagleMemcache: FinagleMemcache, - statsReceiver: StatsReceiver - ): TtlCache[Long, tp.Tweet] = { + val hotkeyCacheClient = new HotKeyMemcacheClient( + proxyClient = memCacheClient, + inProcessCache = new ExpiringLruInProcessCache(ttl = 15.minute, maximumSize = 75000), + statsReceiver = statsReceiver.scope(ScopeName).scope("inProcess") + ) + + val krpcDest = serviceIdentifier.environment.toLowerCase match { + case "prod" => ProdKrpcDest + case _ => DevelKrpcDest + } + + val deferredRpcClient = new DeferredRPC.FinagledClient( + ClientBuilder() + .name("deferredRpc") + .dest(krpcDest) + .requestTimeout(200.milliseconds) + .hostConnectionLimit(3) + .stack(ThriftMux.client.withMutualTls(serviceIdentifier)) + .build() + ) + + val cproxyClient = new cp.Cproxy.FinagledClient( + service = new DeferredThriftService( + deferredrpcService = deferredRpcClient, + target = Target(Datacenter.AllOthers, "cproxy"), + statsReceiver = statsReceiver.scope("deferredThriftService") + ), + stats = statsReceiver.scope("cproxy") + ) + val baseCache: KeyValueTransformingTtlCache[Long, String, tp.Tweet, Array[Byte]] = new KeyValueTransformingTtlCache( - underlyingCache = finagleMemcache, + underlyingCache = new FinagleMemcache(hotkeyCacheClient), transformer = tweetsSerializer, underlyingKey = keyTransformer ) - ObservableTtlCache( + + val observableTtlCache = ObservableTtlCache( underlyingCache = baseCache, statsReceiver = statsReceiver.scope(ScopeName), windowSize = 1000, name = ScopeName ) + + new CproxyTtlCacheWrapper[Long, tp.Tweet]( + underlyingCache = observableTtlCache, + cProxyCache = cproxyClient, + cType = cp.CachePoolType.HomeTweetCoreData, + valueSerializer = tweetsSerializer, + keyTransformer = keyTransformer, + replicationAvailable = Gate.True + ) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TwhinEmbeddingsModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TwhinEmbeddingsModule.scala new file mode 100644 index 000000000..2712fef50 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TwhinEmbeddingsModule.scala @@ -0,0 +1,62 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinRebuildTweetEmbeddingsStore +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinRebuildUserPositiveEmbeddingsStore +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinTweetEmbeddingsStore +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserNegativeEmbeddingsStore +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserPositiveEmbeddingsStore +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinVideoEmbeddingsStore +import com.twitter.home_mixer.store.TwhinEmbeddingsStore +import com.twitter.inject.TwitterModule +import com.twitter.simclusters_v2.{thriftscala => t} +import com.twitter.storehaus.ReadableStore +import javax.inject.Named +import javax.inject.Singleton + +object TwhinEmbeddingsModule extends TwitterModule { + + @Provides + @Singleton + @Named(TwhinTweetEmbeddingsStore) + def providesTwhinTweetEmbeddingFeaturesStore( + twhinEmbeddingsStore: TwhinEmbeddingsStore + ): ReadableStore[Long, t.TwhinTweetEmbedding] = twhinEmbeddingsStore.cachedTweetStore + + @Provides + @Singleton + @Named(TwhinVideoEmbeddingsStore) + def providesTwhinVideoEmbeddingFeaturesStore( + twhinEmbeddingsStore: TwhinEmbeddingsStore + ): ReadableStore[Long, t.TwhinTweetEmbedding] = twhinEmbeddingsStore.cachedVideoStore + + @Provides + @Singleton + @Named(TwhinUserPositiveEmbeddingsStore) + def providesTwhinUserPositiveEmbeddingFeaturesStore( + twhinEmbeddingsStore: TwhinEmbeddingsStore + ): ReadableStore[Long, t.TwhinTweetEmbedding] = twhinEmbeddingsStore.mhUserPositiveStore + + @Provides + @Singleton + @Named(TwhinRebuildUserPositiveEmbeddingsStore) + def providesTwhinRebuildUserPositiveEmbeddingFeatureStore( + twhinEmbeddingStore: TwhinEmbeddingsStore + ): ReadableStore[(Long, Long), t.TwhinTweetEmbedding] = + twhinEmbeddingStore.mhRebuildUserPositiveStore + + @Provides + @Singleton + @Named(TwhinUserNegativeEmbeddingsStore) + def providesTwhinUserNegativeEmbeddingFeaturesStore( + twhinEmbeddingsStore: TwhinEmbeddingsStore + ): ReadableStore[Long, t.TwhinTweetEmbedding] = twhinEmbeddingsStore.mhUserNegativeStore + + @Provides + @Singleton + @Named(TwhinRebuildTweetEmbeddingsStore) + def providesTwhinRebuildTweetEmbeddingFeaturesStore( + twhinEmbeddingsStore: TwhinEmbeddingsStore + ): ReadableStore[(Long, Long), t.TwhinTweetEmbedding] = + twhinEmbeddingsStore.cachedTweetRebuildStore +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/UttTopicModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/UttTopicModule.scala new file mode 100644 index 000000000..0bd04fb01 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/UttTopicModule.scala @@ -0,0 +1,106 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.escherbird.util.uttclient.CacheConfigV2 +import com.twitter.escherbird.util.uttclient.CachedUttClientV2 +import com.twitter.escherbird.util.uttclient.UttClientCacheConfigsV2 +import com.twitter.escherbird.utt.strato.{thriftscala => utt} +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout +import com.twitter.inject.TwitterModule +import com.twitter.inject.annotations.Flag +import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ConfigRepoLocalPath +import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ServiceLocal +import com.twitter.strato.client.Client +import com.twitter.topiclisting.TopicListing +import com.twitter.topiclisting.TopicListingBuilder +import com.twitter.topiclisting.clients.utt.UttClient +import com.twitter.topiclisting.utt.UttLocalization +import com.twitter.topiclisting.utt.UttLocalizationImpl +import com.twitter.tsp.stores.LocalizedUttRecommendableTopicsStore +import com.twitter.tsp.stores.TopicStore +import com.twitter.tsp.stores.UttTopicFilterStore +import com.twitter.util.JavaTimer +import javax.inject.Named +import javax.inject.Singleton + +object UttTopicModule extends TwitterModule { + + private val StatsScope = "UttTopic" + + private val optOutStratoStorePath: String = "interests/optOutInterests" + + private val notInterestedInStorePath: String = "interests/notInterestedTopicsGetter" + + @Provides + @Singleton + def providesTopicListing( + statsReceiver: StatsReceiver, + @Flag(ServiceLocal) isServiceLocal: Boolean, + @Flag(ConfigRepoLocalPath) localConfigRepoPath: String + ): TopicListing = { + val configSourceBasePath = if (isServiceLocal) Some(localConfigRepoPath) else None + new TopicListingBuilder(statsReceiver, configSourceBasePath).build + } + + @Provides + @Singleton + def providesUttLocalization( + @Named(BatchedStratoClientWithModerateTimeout) + stratoClient: Client, + topicListing: TopicListing, + statsReceiver: StatsReceiver + ): UttLocalization = { + // used the same capacity as in currently used client from TSP + val defaultCacheConfigV2 = CacheConfigV2(capacity = 262143) + val uttClientCacheConfigsV2 = UttClientCacheConfigsV2( + getTaxonomyConfig = defaultCacheConfigV2, + getUttTaxonomyConfig = defaultCacheConfigV2, + getLeafIds = defaultCacheConfigV2, + getLeafUttEntities = defaultCacheConfigV2 + ) + + lazy val cachedUttClientV2 = new CachedUttClientV2( + stratoClient = stratoClient, + env = utt.Environment.Prod, + cacheConfigs = uttClientCacheConfigsV2, + statsReceiver = statsReceiver.scope("CachedUttClient") + ) + val uttClient = new UttClient(cachedUttClientV2, statsReceiver) + + new UttLocalizationImpl(topicListing, uttClient, statsReceiver.scope(StatsScope)) + } + + @Provides + @Singleton + def providesUttTopicFilterStore( + @Named(BatchedStratoClientWithModerateTimeout) + stratoClient: Client, + uttLocalization: UttLocalization, + topicListing: TopicListing, + statsReceiver: StatsReceiver + ): UttTopicFilterStore = { + val userOptOutTopicsStore = + TopicStore.userOptOutTopicStore(stratoClient, optOutStratoStorePath)( + statsReceiver.scope("interests_opt_out_store")) + val explicitFollowingTopicsStore = + TopicStore.explicitFollowingTopicStore(stratoClient)( + statsReceiver.scope("explicit_following_interests_store")) + val userNotInterestedInTopicsStore = + TopicStore.notInterestedInTopicsStore(stratoClient, notInterestedInStorePath)( + statsReceiver.scope("not_interested_in_store")) + val localizedUttRecommendableTopicsStore = new LocalizedUttRecommendableTopicsStore( + uttLocalization) + val timer = new JavaTimer(isDaemon = true) + + new UttTopicFilterStore( + topicListing = topicListing, + userOptOutTopicsStore = userOptOutTopicsStore, + explicitFollowingTopicsStore = explicitFollowingTopicsStore, + notInterestedTopicsStore = userNotInterestedInTopicsStore, + localizedUttRecommendableTopicsStore = localizedUttRecommendableTopicsStore, + timer = timer, + stats = statsReceiver.scope("UttTopicFilterStore") + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/VideoEmbeddingModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/VideoEmbeddingModule.scala new file mode 100644 index 000000000..85c029085 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/VideoEmbeddingModule.scala @@ -0,0 +1,22 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.home_mixer.param.HomeMixerInjectionNames.VideoEmbeddingMHStore +import com.twitter.home_mixer.store.VideoEmbeddingMHStore +import com.twitter.inject.TwitterModule +import com.twitter.media_understanding.video_summary.thriftscala.VideoEmbedding +import com.twitter.storehaus.ReadableStore +import javax.inject.Named +import javax.inject.Singleton + +object VideoEmbeddingModule extends TwitterModule { + + @Provides + @Singleton + @Named(VideoEmbeddingMHStore) + def providesVideoEmbeddingMHStore( + videoEmbeddingMHStore: VideoEmbeddingMHStore + ): ReadableStore[Long, VideoEmbedding] = { + videoEmbeddingMHStore.videoEmbeddingMHStore + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.bazel index 25e9a2e31..b4ff2a9b8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.bazel @@ -4,6 +4,7 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala index 6fbd28fce..0ce7c4b34 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala @@ -1,5 +1,36 @@ package com.twitter.home_mixer.param +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableClipEmbeddingFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableClipEmbeddingMediaUnderstandingFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableDedupClusterId88FeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableDedupClusterIdFeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableGeoduckAuthorLocationHydatorParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableGrokVideoMetadataFeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableImmersiveClientActionsClipEmbeddingQueryFeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableImmersiveClientActionsQueryFeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableOnPremRealGraphQueryFeatures +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableRealGraphQueryFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableRealGraphViewerRelatedUsersFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableSimclustersSparseTweetFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTransformerPostEmbeddingJointBlueFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetLanguageFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetTextTokensEmbeddingFeatureScribingParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetVideoAggregatedWatchTimeFeatureScribingParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetypieContentFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetypieContentMediaEntityFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildTweetFeaturesOnlineParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildUserEngagementFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildUserPositiveFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinTweetFeaturesOnlineParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinUserNegativeFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinUserPositiveFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinVideoFeaturesOnlineParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinVideoFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableUserFavAvgTextEmbeddingsQueryFeatureParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableUserHistoryTransformerJointBlueEmbeddingFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableViewCountFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring import com.twitter.home_mixer.param.HomeGlobalParams._ import com.twitter.product_mixer.core.functional_component.configapi.registry.GlobalParamConfig import javax.inject.Inject @@ -12,23 +43,248 @@ import javax.inject.Singleton @Singleton class HomeGlobalParamConfig @Inject() () extends GlobalParamConfig { + override val booleanDeciderOverrides = Seq( + EnableServedCandidateFeatureKeysKafkaPublishingParam, + FeatureHydration.EnableSimClustersSimilarityFeaturesDeciderParam, + FeatureHydration.EnableTweetypieContentFeaturesDeciderParam, + FeatureHydration.EnableVideoSummaryEmbeddingFeatureDeciderParam, + FeatureHydration.EnableVideoClipEmbeddingFeatureHydrationDeciderParam, + FeatureHydration.EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam, + FeatureHydration.EnableVideoClipEmbeddingMediaUnderstandingFeatureHydrationDeciderParam, + EnableCommonFeaturesDataRecordCopyDuringPldrConversionParam, + Scoring.UseSecondaryNaviClusterParam, + Scoring.UseGPUNaviClusterTestUsersParam + ) + + override val boundedDoubleDeciderOverrides = Seq(Scoring.NaviGPUBatchSizeParam) + + override val stringFSOverrides = + Seq( + Scoring.ModelNameParam, + Scoring.ModelIdParam, + Scoring.ProdModelIdParam, + PostFeedbackPromptTitleParam, + PostFeedbackPromptPositiveParam, + PostFeedbackPromptNegativeParam, + PostFeedbackPromptNeutralParam, + ) + override val booleanFSOverrides = Seq( AdsDisableInjectionBasedOnUserRoleParam, + EnableTweetEntityServiceMigrationParam, + EnableTweetEntityServiceVisibilityMigrationParam, EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - EnableImpressionBloomFilter, - EnableNahFeedbackInfoParam, + EnableSSPAdsBrandSafetySettingsFeatureHydratorParam, + EnableDebugString, + EnablePersistenceDebug, + EnableLargeEmbeddingsFeatureHydrationParam, EnableNewTweetsPillAvatarsParam, + EnableOnPremRealGraphQueryFeatures, + EnableRealGraphQueryFeaturesParam, + EnableRealGraphViewerRelatedUsersFeaturesParam, EnableScribeServedCandidatesParam, EnableSendScoresToClient, + EnableSimclustersSparseTweetFeaturesParam, + EnableCommunitiesContextParam, EnableSocialContextParam, + EnableTwhinRebuildUserEngagementFeaturesParam, + EnableTwhinRebuildUserPositiveFeaturesParam, + EnableTwhinVideoFeaturesParam, + EnableTwhinUserNegativeFeaturesParam, + EnableTwhinUserPositiveFeaturesParam, + EnableTwhinVideoFeaturesOnlineParam, + EnableTwhinTweetFeaturesOnlineParam, + EnableTwhinRebuildTweetFeaturesOnlineParam, + EnableBasketballContextFeatureHydratorParam, + EnablePostContextFeatureHydratorParam, + EnableTransformerPostEmbeddingJointBlueFeaturesParam, + EnableClipEmbeddingFeaturesParam, + EnableClipEmbeddingMediaUnderstandingFeaturesParam, + EnableUserHistoryTransformerJointBlueEmbeddingFeaturesParam, + EnableLandingPage, + EnableTenSecondsLogicForVQV, + EnableImmersiveVQV, + EnableExploreSimclustersLandingPage, + EnableTweetLanguageFeaturesParam, + EnableTweetypieContentFeaturesParam, + EnableTweetypieContentMediaEntityFeaturesParam, + EnableViewCountFeaturesParam, + ListMandarinTweetsParams.ListMandarinTweetsEnable, + Scoring.EnableNoNegHeuristicParam, + Scoring.EnableNegSectionRankingParam, + Scoring.EnableBinarySchemeForVQVParam, + Scoring.EnableBinarySchemeForDwellParam, + Scoring.EnableDwellOrVQVParam, + Scoring.UseRealtimeNaviClusterParam, + Scoring.UseGPUNaviClusterParam, + Scoring.RequestNormalizedScoresParam, + Scoring.AddNoiseInWeightsPerLabel, + Scoring.EnableDailyFrozenNoisyWeights, + Scoring.TwhinDiversityRescoringParam, + Scoring.CategoryDiversityRescoringParam, + Scoring.UseVideoNaviClusterParam, + Scoring.NormalizedNegativeHead, + Scoring.UseWeightForNegHeadParam, + Scoring.ConstantNegativeHead, + Scoring.UseProdInPhoenixParams.EnableProdDwellForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdFavForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdGoodClickV1ForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdGoodClickV2ForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdNegForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdProfileClickForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdReplyForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdRetweetForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdShareForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdVQVForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdOpenLinkForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdScreenshotForPhoenixParam, + Scoring.UseProdInPhoenixParams.EnableProdBookmarkForPhoenixParam, + EnableUserFavAvgTextEmbeddingsQueryFeatureParam, + EnableTweetTextTokensEmbeddingFeatureScribingParam, + EnableTweetVideoAggregatedWatchTimeFeatureScribingParam, + EnableImmersiveClientActionsQueryFeatureHydrationParam, + EnableImmersiveClientActionsClipEmbeddingQueryFeatureHydrationParam, + EnableGrokVideoMetadataFeatureHydrationParam, + EnableDedupClusterIdFeatureHydrationParam, + EnableDedupClusterId88FeatureHydrationParam, + EnableServedFilterAllRequests, + EnablePinnedTweetsCarouselParam, + EnablePostDetailsNegativeFeedbackParam, + EnablePostFeedbackParam, + EnablePostFollowupParam, + EnableSlopFilter, + EnableNsfwFilter, + EnableSoftNsfwFilter, + EnableGrokGoreFilter, + EnableMinVideoDurationFilter, + EnableMaxVideoDurationFilter, + EnableGrokSpamFilter, + EnableGrokViolentFilter, + EnableClusterBasedDedupFilter, + EnableCountryFilter, + EnableRegionFilter, + EnableHasMultipleMediaFilter, + EnableClusterBased88DedupFilter, + EnableNoClusterFilter, + EnableSlopFilterLowSignalUsers, + EnableSlopFilterEligibleUserStateParam, + EnableGrokAnnotations, + EnableTopicBasedRealTimeAggregateFeatureHydratorParam, + EnableTopicCountryBasedRealTimeAggregateFeatureHydratorParam, + EnableTopicEdgeAggregateFeatureHydratorParam, + EnableAdditionalChildFeedbackParam, + EnableGeoduckAuthorLocationHydatorParam, + EnableTweetRTAMhOnlyParam, + EnableTweetRTAMhFallbackParam, + EnableTweetCountryRTAMhOnlyParam, + EnableTweetCountryRTAMhFallbackParam, + EnableUserRTAMhOnlyParam, + EnableUserRTAMhFallbackParam, + EnableUserAuthorRTAMhOnlyParam, + EnableUserAuthorRTAMhFallbackParam, + EnableBlockMuteReportChildFeedbackParam, + EnablePhoenixScorerParam, + EnableUserActionsShadowScribeParam, ) override val boundedIntFSOverrides = Seq( MaxNumberReplaceInstructionsParam, TimelinesPersistenceStoreMaxEntriesPerClient, + ExcludeServedTweetIdsNumberParam, + IsSelectedByHeavyRankerCountParam, + SlopMinFollowers, + UserActionsMaxCount, + PhoenixTimeoutInMsParam, ) + override val boundedLongFSOverrides = + Seq( + DedupHistoricalEventsTimeWindowParam, + MinVideoDurationThresholdParam, + MaxVideoDurationThresholdParam) override val boundedDoubleFSOverrides = Seq( - ImpressionBloomFilterFalsePositiveRateParam + SlopMaxScore, + PostFeedbackThresholdParam, + PostFollowupThresholdParam, + // Model Weights + Scoring.ModelWeights.BookmarkParam, + Scoring.ModelWeights.Dwell0Param, + Scoring.ModelWeights.Dwell1Param, + Scoring.ModelWeights.Dwell2Param, + Scoring.ModelWeights.Dwell3Param, + Scoring.ModelWeights.Dwell4Param, + Scoring.ModelWeights.DwellParam, + Scoring.ModelWeights.FavParam, + Scoring.ModelWeights.GoodClickParam, + Scoring.ModelWeights.GoodClickV1Param, + Scoring.ModelWeights.GoodClickV2Param, + Scoring.ModelWeights.GoodProfileClickParam, + Scoring.ModelWeights.NegativeFeedbackV2Param, + Scoring.ModelWeights.OpenLinkParam, + Scoring.ModelWeights.ProfileDwelledParam, + Scoring.ModelWeights.ReplyEngagedByAuthorParam, + Scoring.ModelWeights.ReplyParam, + Scoring.ModelWeights.ReportParam, + Scoring.ModelWeights.RetweetParam, + Scoring.ModelWeights.ScreenshotParam, + Scoring.ModelWeights.ShareMenuClickParam, + Scoring.ModelWeights.ShareParam, + Scoring.ModelWeights.StrongNegativeFeedbackParam, + Scoring.ModelWeights.TweetDetailDwellParam, + Scoring.ModelWeights.VideoPlayback50Param, + Scoring.ModelWeights.VideoQualityViewImmersiveParam, + Scoring.ModelWeights.VideoQualityViewParam, + Scoring.ModelWeights.VideoQualityWatchParam, + Scoring.ModelWeights.VideoWatchTimeMsParam, + Scoring.ModelWeights.WeakNegativeFeedbackParam, + // Model Biases + Scoring.ModelBiases.VideoQualityViewParam, + Scoring.ModelBiases.VideoQualityViewImmersiveParam, + Scoring.ModelBiases.VideoQualityWatchParam, + Scoring.NoisyWeightAlphaParam, + Scoring.NoisyWeightBetaParam, + Scoring.NegativeScoreConstantFilterThresholdParam, + Scoring.NegativeScoreNormFilterThresholdParam, + Scoring.RequestRankDecayFactorParam, + Scoring.ScoreThresholdForVQVParam, + Scoring.ScoreThresholdForDwellParam, + Scoring.BinarySchemeConstantForVQVParam, + Scoring.ImpressedMediaClusterBasedRescoringParam, + // ModelDebiases + Scoring.ModelDebiases.FavParam, + Scoring.ModelDebiases.ReplyParam, + Scoring.ModelDebiases.RetweetParam, + Scoring.ModelDebiases.GoodClickV1Param, + Scoring.ModelDebiases.GoodClickV2Param, + Scoring.ModelDebiases.GoodProfileClickParam, + Scoring.ModelDebiases.ReplyEngagedByAuthorParam, + Scoring.ModelDebiases.VideoQualityViewParam, + Scoring.ModelDebiases.VideoQualityViewImmersiveParam, + Scoring.ModelDebiases.NegativeFeedbackV2Param, + Scoring.ModelDebiases.BookmarkParam, + Scoring.ModelDebiases.ShareParam, + Scoring.ModelDebiases.DwellParam, + Scoring.ModelDebiases.VideoQualityWatchParam, + Scoring.ModelDebiases.VideoWatchTimeMsParam + ) + + override val longSetFSOverrides = Seq( + RateLimitTestIdsParam, + BasketballTeamAccountIdsParam, + Scoring.AuthorListForDataCollectionParam + ) + + override val boundedDurationFSOverrides = Seq( + FeedbackFatigueFilteringDurationParam, + ExcludeServedTweetIdsDurationParam, + ExcludeServedAuthorIdsDurationParam + ) + + override val enumFSOverrides = Seq( + PhoenixInferenceClusterParam + ) + + override val longSeqFSOverrides = Seq( + ListMandarinTweetsParams.ListMandarinTweetsLists ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala index f19817bc9..f48870bb6 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala @@ -1,7 +1,15 @@ package com.twitter.home_mixer.param +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.param.decider.DeciderKey +import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.HasDurationConversion +import com.twitter.timelines.configapi.decider.BooleanDeciderParam +import com.twitter.timelines.configapi.decider.DeciderBoundedParam +import com.twitter.util.Duration /** * Instantiate Params that do not relate to a specific product. @@ -16,7 +24,22 @@ object HomeGlobalParams { * and should NOT be used for other purposes. */ object AdsDisableInjectionBasedOnUserRoleParam - extends FSParam("home_mixer_ads_disable_injection_based_on_user_role", false) + extends FSParam( + name = "home_mixer_ads_disable_injection_based_on_user_role", + default = false + ) + + object EnableTweetEntityServiceMigrationParam + extends FSParam[Boolean]( + name = "home_mixer_enable_tweet_entity_service_migration", + default = false + ) + + object EnableTweetEntityServiceVisibilityMigrationParam + extends FSParam[Boolean]( + name = "home_mixer_enable_tweet_entity_service_visibility_migration", + default = false + ) object EnableSendScoresToClient extends FSParam[Boolean]( @@ -24,18 +47,24 @@ object HomeGlobalParams { default = false ) - object EnableNahFeedbackInfoParam + object EnableDebugString extends FSParam[Boolean]( - name = "home_mixer_enable_nah_feedback_info", + name = "home_mixer_enable_debug_string", + default = false + ) + + object EnablePersistenceDebug + extends FSParam[Boolean]( + name = "home_mixer_enable_persistence_debug", default = false ) object MaxNumberReplaceInstructionsParam extends FSBoundedParam[Int]( name = "home_mixer_max_number_replace_instructions", - default = 100, - min = 0, - max = 200 + default = 10, + min = 1, + max = 20 ) object TimelinesPersistenceStoreMaxEntriesPerClient @@ -55,6 +84,12 @@ object HomeGlobalParams { object EnableSocialContextParam extends FSParam[Boolean]( name = "home_mixer_enable_social_context", + default = false + ) + + object EnableCommunitiesContextParam + extends FSParam[Boolean]( + name = "home_mixer_enable_communities_context", default = true ) @@ -64,23 +99,1381 @@ object HomeGlobalParams { default = true ) - object EnableImpressionBloomFilter + object EnableBasketballContextFeatureHydratorParam extends FSParam[Boolean]( - name = "home_mixer_enable_impression_bloom_filter", + name = "home_mixer_enable_basketball_context_feature_hydrator", default = false ) - object ImpressionBloomFilterFalsePositiveRateParam - extends FSBoundedParam[Double]( - name = "home_mixer_impression_bloom_filter_false_positive_rate", - default = 0.005, - min = 0.001, - max = 0.01 + object EnablePostContextFeatureHydratorParam + extends FSParam[Boolean]( + name = "home_mixer_enable_post_context_feature_hydrator", + default = false + ) + + object BasketballTeamAccountIdsParam + extends FSParam[Set[Long]]( + name = "home_mixer_basketball_team_account_ids", + default = Set() + ) + + object EnableSSPAdsBrandSafetySettingsFeatureHydratorParam + extends FSParam[Boolean]( + name = "home_mixer_enable_ssp_ads_brand_safety_settings_feature_hydrator", + default = true + ) + + object ExcludeServedTweetIdsNumberParam + extends FSBoundedParam[Int]( + name = "home_mixer_exclude_served_tweet_ids_number", + default = 100, + min = 0, + max = 100 + ) + + object ExcludeServedTweetIdsDurationParam + extends FSBoundedParam[Duration]( + "home_mixer_exclude_served_tweet_ids_in_minutes", + default = 10.minutes, + min = 1.minute, + max = 60.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object ExcludeServedAuthorIdsDurationParam + extends FSBoundedParam[Duration]( + "home_mixer_exclude_served_author_ids_in_minutes", + default = 60.minutes, + min = 1.minute, + max = 60.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object EnableServedFilterAllRequests + extends FSParam[Boolean]( + name = "home_mixer_enable_served_filter_all_requests", + default = false ) object EnableScribeServedCandidatesParam extends FSParam[Boolean]( name = "home_mixer_served_tweets_enable_scribing", + default = false + ) + + object EnableServedCandidateFeatureKeysKafkaPublishingParam + extends BooleanDeciderParam( + decider = DeciderKey.EnableServedCandidateFeatureKeysKafkaPublishing) + + object RateLimitTestIdsParam + extends FSParam[Set[Long]]( + name = "home_mixer_rate_limit_test_ids", + default = Set.empty + ) + + object IsSelectedByHeavyRankerCountParam + extends FSBoundedParam[Int]( + name = "home_mixer_is_selected_by_heavy_ranker_count", + default = 100, + min = 0, + max = 2000 + ) + + object EnableAdditionalChildFeedbackParam + extends FSParam[Boolean]( + name = "home_mixer_enable_additional_child_feedback", + default = false + ) + + object EnableBlockMuteReportChildFeedbackParam + extends FSParam[Boolean]( + name = "home_mixer_enable_block_mute_report_child_feedback", + default = false + ) + + object ListMandarinTweetsParams { + object ListMandarinTweetsEnable + extends FSParam[Boolean]( + name = "home_mixer_mandarin_list_tweets_enabled", + default = false + ) + + object ListMandarinTweetsLists + extends FSParam[Seq[Long]]( + name = "home_mixer_mandarin_tweets_lists", + default = Seq.empty + ) + } + + object FeatureHydration { + object EnableLargeEmbeddingsFeatureHydrationParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_large_embeddings", + default = false + ) + + object EnableSimClustersSimilarityFeaturesDeciderParam + extends BooleanDeciderParam( + decider = DeciderKey.EnableSimClustersSimilarityFeatureHydration + ) + + object EnableOnPremRealGraphQueryFeatures + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_on_prem_real_graph_query_features", + default = false + ) + + object EnableRealGraphQueryFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_real_graph_query_features", + default = false + ) + + object EnableRealGraphViewerRelatedUsersFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_real_graph_viewer_related_users_features", + default = false + ) + + object EnableSimclustersSparseTweetFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_simclusters_sparse_tweet_features", + default = false + ) + + object EnableTwhinUserPositiveFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_twhin_user_positive_features", + default = false + ) + + object EnableTwhinVideoFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_twhin_video_features", + default = false + ) + + object EnableTwhinUserNegativeFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_twhin_user_negative_features", + default = false + ) + + object EnableTwhinVideoFeaturesOnlineParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_twhin_video_online_features", + default = false + ) + + object EnableTwhinRebuildUserEngagementFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_twhin_rebuild_user_engagement_features", + default = false + ) + + object EnableTwhinRebuildUserPositiveFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_twhin_rebuild_user_positive_features", + default = false + ) + + object EnableClipEmbeddingFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_clip_embedding_features", + default = false + ) + + object EnableClipEmbeddingMediaUnderstandingFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_clip_embedding_media_understanding_features", + default = false + ) + + object EnableUserHistoryTransformerJointBlueEmbeddingFeaturesParam + extends FSParam[Boolean]( + name = + "home_mixer_feature_hydration_enable_user_history_transformer_joint_blue_embedding_features", + default = false + ) + + object EnableTweetLanguageFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_tweet_language_features", + default = false + ) + + object EnableTwhinTweetFeaturesOnlineParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_twhin_tweet_online_features", + default = false + ) + + object EnableTwhinRebuildTweetFeaturesOnlineParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_twhin_rebuild_tweet_online_features", + default = false + ) + + object EnableTransformerPostEmbeddingJointBlueFeaturesParam + extends FSParam[Boolean]( + name = + "home_mixer_feature_hydration_enable_transformer_post_embedding_features_joint_blue", + default = false + ) + + object EnableTweetypieContentFeaturesDeciderParam + extends BooleanDeciderParam( + decider = DeciderKey.EnableTweetypieContentFeatures + ) + + object EnableTweetypieContentFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_tweetypie_content_features", + default = true + ) + + object EnableTweetypieContentMediaEntityFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_tweetypie_content_media_entity_features", + default = true + ) + + object EnableUserFavAvgTextEmbeddingsQueryFeatureParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_user_fav_avg_text_embeddings_query_feature", + default = false + ) + + object EnableTweetTextTokensEmbeddingFeatureScribingParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_tweet_text_tokens_embedding_feature_scribing", + default = false + ) + + object EnableTweetVideoAggregatedWatchTimeFeatureScribingParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_tweet_video_aggregated_watch_time", + default = false + ) + + object EnableImmersiveClientActionsQueryFeatureHydrationParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_immersive_client_actions", + default = false + ) + + object EnableImmersiveClientActionsClipEmbeddingQueryFeatureHydrationParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_immersive_client_actions_clip_embedding", + default = false + ) + + object EnableGrokVideoMetadataFeatureHydrationParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_grok_video_metadata", + default = false + ) + + object EnableDedupClusterIdFeatureHydrationParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_dedup_cluster_id", + default = false + ) + + object EnableDedupClusterId88FeatureHydrationParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_dedup_cluster_id_88", + default = false + ) + + object EnableGeoduckAuthorLocationHydatorParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_geoduck_author_location_hydrator", + default = false + ) + + object EnableViewCountFeaturesParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_view_count_features", + default = false + ) + + object EnableVideoSummaryEmbeddingFeatureDeciderParam + extends BooleanDeciderParam( + decider = DeciderKey.EnableVideoSummaryEmbeddingHydration + ) + + object EnableVideoClipEmbeddingFeatureHydrationDeciderParam + extends BooleanDeciderParam( + decider = DeciderKey.EnableVideoClipEmbeddingHydration + ) + + object EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam + extends BooleanDeciderParam( + decider = + DeciderKey.EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam + ) + + object EnableVideoClipEmbeddingMediaUnderstandingFeatureHydrationDeciderParam + extends BooleanDeciderParam( + decider = DeciderKey.EnableVideoClipEmbeddingMediaUnderstandingHydration + ) + } + + object Scoring { + + object AuthorListForDataCollectionParam + extends FSParam[Set[Long]]( + name = "home_mixer_author_list_for_data_collection", + default = Set.empty[Long] + ) + + object ModelNameParam + extends FSParam[String]( + name = "home_mixer_model_name", + default = "" + ) + + object ImpressedMediaClusterBasedRescoringParam + extends FSBoundedParam[Double]( + name = "home_mixer_impressed_media_cluster_based_rescoring", + default = 0.0, + min = 0.0, + max = 0.2 + ) + + object ImpressedImageClusterBasedRescoringParam + extends FSBoundedParam[Double]( + name = "home_mixer_impressed_image_cluster_based_rescoring", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object ModelIdParam + extends FSParam[String]( + name = "home_mixer_model_id", + default = "Home" + ) + + object ProdModelIdParam + extends FSParam[String]( + name = "home_mixer_model_prod_model_id", + default = "Home" + ) + + object UseRealtimeNaviClusterParam + extends FSParam[Boolean]( + name = "home_mixer_model_use_realtime_navi_cluster", + default = false + ) + + object UseGPUNaviClusterParam + extends FSParam[Boolean]( + name = "home_mixer_model_use_gpu_navi_cluster", + default = false + ) + + object UseSecondaryNaviClusterParam + extends BooleanDeciderParam(decider = DeciderKey.EnableSecondaryNaviRecapCluster) + + object UseGPUNaviClusterTestUsersParam + extends BooleanDeciderParam(decider = DeciderKey.EnableGPUNaviRecapClusterTestUsers) + + object UseVideoNaviClusterParam + extends FSParam[Boolean]("home_mixer_model_use_video_navi_cluster", false) + + object NaviGPUBatchSizeParam + extends DeciderBoundedParam[Double]( + decider = DeciderKey.NaviGPUClusterRequestBatchSize, + default = 1800.0, + min = 0.0, + max = 10000.0 + ) + + object AddNoiseInWeightsPerLabel + extends FSParam[Boolean]( + name = "home_mixer_add_noise_in_weights_per_label", + default = false + ) + + object EnableDailyFrozenNoisyWeights + extends FSParam[Boolean]( + name = "home_mixer_enable_daily_frozen_weights", + default = false + ) + + object NoisyWeightAlphaParam + extends FSBoundedParam[Double]( + name = "home_mixer_noisy_weight_alpha_param", + default = 2, + min = 0.0, + max = 10.0 + ) + + object NoisyWeightBetaParam + extends FSBoundedParam[Double]( + name = "home_mixer_noisy_weight_beta_param", + default = 2, + min = 0.0, + max = 10.0 + ) + object NegativeScoreConstantFilterThresholdParam + extends FSBoundedParam[Double]( + name = "home_mixer_negative_score_constant_filter_threshold", + default = 1e-3, + min = 0, + max = 1 + ) + + object NegativeScoreNormFilterThresholdParam + extends FSBoundedParam[Double]( + name = "home_mixer_negative_score_norm_filter_threshold", + default = 0.15, + min = 0, + max = 1 + ) + + object RequestNormalizedScoresParam + extends FSParam[Boolean]( + name = "home_mixer_request_normalized_scores", + default = false + ) + + object NormalizedNegativeHead + extends FSParam[Boolean]( + name = "home_mixer_normalized_negative_head", + default = false + ) + + object UseWeightForNegHeadParam + extends FSParam[Boolean]( + name = "home_mixer_use_weight_for_neg_head", + default = false + ) + + object ConstantNegativeHead + extends FSParam[Boolean]( + name = "home_mixer_constant_negative_head", + default = false + ) + + object EnableNoNegHeuristicParam + extends FSParam[Boolean]( + name = "home_mixer_no_neg_heuristic", + default = false + ) + + object EnableNegSectionRankingParam + extends FSParam[Boolean]( + name = "home_mixer_neg_section_ranking", + default = false + ) + + object RequestRankDecayFactorParam + extends FSBoundedParam[Double]( + name = "home_mixer_request_rank_decay_factor", + default = 0.95, + min = 0, + max = 1 + ) + + object ScoreThresholdForVQVParam + extends FSBoundedParam[Double]( + name = "home_mixer_score_threshold_for_vqv", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object ScoreThresholdForDwellParam + extends FSBoundedParam[Double]( + name = "home_mixer_score_threshold_for_dwell", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object EnableBinarySchemeForVQVParam + extends FSParam[Boolean]( + name = "home_mixer_enable_binary_scheme_for_vqv", + default = false + ) + + object BinarySchemeConstantForVQVParam + extends FSBoundedParam[Double]( + name = "home_mixer_constant_binary_scheme_for_vqv", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object EnableBinarySchemeForDwellParam + extends FSParam[Boolean]( + name = "home_mixer_enable_binary_scheme_for_dwell", + default = false + ) + + object EnableDwellOrVQVParam + extends FSParam[Boolean]( + name = "home_mixer_enable_dwell_or_video_watch_time", + default = false + ) + + object TwhinDiversityRescoringParam + extends FSParam[Boolean]( + name = "home_mixer_twhin_diversity_rescoring", + default = false + ) + + object CategoryDiversityRescoringParam + extends FSParam[Boolean]( + name = "home_mixer_category_diversity_rescoring", + default = false + ) + + object ModelBiases { + object VideoQualityViewParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_bias_video_quality_viewed", + default = 0.0, + min = 0.0, + max = 100.0 + ) + + object VideoQualityViewImmersiveParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_bias_video_quality_viewed_immersive", + default = 0.0, + min = 0.0, + max = 100.0 + ) + + object VideoQualityWatchParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_bias_video_quality_watched", + default = 0.0, + min = 0.0, + max = 100.0 + ) + } + + object ModelDebiases { + + object FavParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_fav", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object RetweetParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_retweet", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object ReplyParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_reply", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object DwellParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_dwell", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object GoodProfileClickParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_good_profile_click", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object VideoWatchTimeMsParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_video_watch_time_ms", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object VideoQualityViewParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_video_quality_viewed", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object VideoQualityViewImmersiveParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_video_quality_viewed_immersive", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object ReplyEngagedByAuthorParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_reply_engaged_by_author", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object GoodClickV1Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_good_click_v1", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object GoodClickV2Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_good_click_v2", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object BookmarkParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_bookmark", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object ShareParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_share", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object NegativeFeedbackV2Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_negative_feedback_v2", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object VideoQualityWatchParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_debias_video_quality_watched", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + } + + object ModelWeights { + + object FavParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_fav", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object RetweetParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_retweet", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object ReplyParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_reply", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object GoodProfileClickParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_good_profile_click", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object VideoPlayback50Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_video_playback50", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object VideoQualityViewParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_video_quality_viewed", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object VideoQualityViewImmersiveParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_video_quality_viewed_immersive", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object ReplyEngagedByAuthorParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_reply_engaged_by_author", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object GoodClickParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_good_click", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object GoodClickV1Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_good_click_v1", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object GoodClickV2Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_good_click_v2", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object TweetDetailDwellParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_tweet_detail_dwell", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object ProfileDwelledParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_profile_dwelled", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object BookmarkParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_bookmark", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object ShareParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_share", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object ShareMenuClickParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_share_menu_click", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object NegativeFeedbackV2Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_negative_feedback_v2", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object ReportParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_report", + default = 0.0, + min = -20000.0, + max = 0.0 + ) + + object WeakNegativeFeedbackParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_weak_negative_feedback", + default = 0.0, + min = -1000.0, + max = 0.0 + ) + + object StrongNegativeFeedbackParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_strong_negative_feedback", + default = 0.0, + min = -1000.0, + max = 0.0 + ) + + object DwellParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_dwell", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object OpenLinkParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_open_link", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object ScreenshotParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_screenshot", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + object VideoWatchTimeMsParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_video_watch_time_ms", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + + // Categorical Dwell Params + object Dwell0Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_dwell_0", + default = 0.0, + min = 0.0, + max = 1000.0 + ) + + object Dwell1Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_dwell_1", + default = 0.0, + min = 0.0, + max = 1000.0 + ) + + object Dwell2Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_dwell_2", + default = 0.0, + min = 0.0, + max = 1000.0 + ) + + object Dwell3Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_dwell_3", + default = 0.0, + min = 0.0, + max = 1000.0 + ) + + object Dwell4Param + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_dwell_4", + default = 0.0, + min = 0.0, + max = 1000.0 + ) + + object VideoQualityWatchParam + extends FSBoundedParam[Double]( + name = "home_mixer_model_weight_video_quality_watched", + default = 0.0, + min = -10000.0, + max = 10000.0 + ) + } + + object UseProdInPhoenixParams { + object EnableProdFavForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_fav_for_phoenix", + default = false + ) + + object EnableProdReplyForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_reply_for_phoenix", + default = false + ) + + object EnableProdShareForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_share_for_phoenix", + default = false + ) + + object EnableProdRetweetForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_retweet_for_phoenix", + default = false + ) + + object EnableProdVQVForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_vqv_for_phoenix", + default = false + ) + + object EnableProdDwellForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_dwell_for_phoenix", + default = false + ) + + object EnableProdNegForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_neg_for_phoenix", + default = false + ) + + object EnableProdProfileClickForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_profile_click_for_phoenix", + default = false + ) + + object EnableProdGoodClickV1ForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_good_click_v1_for_phoenix", + default = false + ) + + object EnableProdGoodClickV2ForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_good_click_v2_for_phoenix", + default = false + ) + + object EnableProdOpenLinkForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_open_link_for_phoenix", + default = false + ) + + object EnableProdScreenshotForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_screenshot_for_phoenix", + default = false + ) + + object EnableProdBookmarkForPhoenixParam + extends FSParam[Boolean]( + name = "home_mixer_enable_prod_bookmark_for_phoenix", + default = false + ) + } + } + + object EnableTenSecondsLogicForVQV + extends FSParam[Boolean]( + name = "home_mixer_enable_ten_seconds_logic_for_vqv", default = true ) + + object EnableImmersiveVQV + extends FSParam[Boolean]( + name = "home_mixer_enable_immersive_vqv", + default = false + ) + object EnableLandingPage + extends FSParam[Boolean]( + name = "home_mixer_enable_landing_page", + default = false + ) + + object EnableExploreSimclustersLandingPage + extends FSParam[Boolean]( + name = "home_mixer_enable_explore_simclusters_landing_page", + default = false + ) + + object EnableTopicBasedRealTimeAggregateFeatureHydratorParam + extends FSParam[Boolean]( + name = "home_mixer_enable_topic_based_real_time_aggregate_feature_hydrator_param", + default = true + ) + + object EnableTopicCountryBasedRealTimeAggregateFeatureHydratorParam + extends FSParam[Boolean]( + name = "home_mixer_enable_topic_country_based_real_time_aggregate_feature_hydrator_param", + default = true + ) + + object EnableTopicEdgeAggregateFeatureHydratorParam + extends FSParam[Boolean]( + name = "home_mixer_enable_topic_edge_aggregate_feature_hydrator_param", + default = true + ) + + object FeedbackFatigueFilteringDurationParam + extends FSBoundedParam[Duration]( + name = "home_mixer_feedback_fatigue_filtering_duration_in_days", + default = 14.days, + min = 0.days, + max = 100.days + ) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromDays + } + + object EnableCommonFeaturesDataRecordCopyDuringPldrConversionParam + extends BooleanDeciderParam( + decider = DeciderKey.EnableCommonFeaturesDataRecordCopyDuringPldrConversion) + + object EnablePinnedTweetsCarouselParam + extends FSParam[Boolean]( + name = "home_mixer_enable_pinned_tweets_carousel", + default = false + ) + + object EnablePostFeedbackParam + extends FSParam[Boolean]( + name = "home_mixer_enable_post_feedback", + default = false + ) + + object PostFeedbackThresholdParam + extends FSBoundedParam[Double]( + name = "home_mixer_post_feedback_threshold", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object PostFeedbackPromptTitleParam + extends FSParam[String]( + name = "home_mixer_post_feedback_prompt_title", + default = "Are you interested in this post?" + ) + + object PostFeedbackPromptPositiveParam + extends FSParam[String]( + name = "home_mixer_post_feedback_prompt_positive", + default = "Yes" + ) + + object PostFeedbackPromptNegativeParam + extends FSParam[String]( + name = "home_mixer_post_feedback_prompt_negative", + default = "No" + ) + + object PostFeedbackPromptNeutralParam + extends FSParam[String]( + name = "home_mixer_post_feedback_prompt_neutral", + default = "Not sure" + ) + + object EnablePostFollowupParam + extends FSParam[Boolean]( + name = "home_mixer_enable_post_followup", + default = false + ) + + object EnablePostDetailsNegativeFeedbackParam + extends FSParam[Boolean]( + name = "home_mixer_enable_post_details_negative_feedback", + default = false + ) + + object PostFollowupThresholdParam + extends FSBoundedParam[Double]( + name = "home_mixer_post_followup_threshold", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object EnableSlopFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_slop_filter", + default = false + ) + + object EnableNsfwFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_nsfw_filter", + default = false + ) + + object EnableSoftNsfwFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_soft_nsfw_filter", + default = false + ) + + object EnableGrokSpamFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_grok_spam_filter", + default = false + ) + + object EnableGrokViolentFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_grok_violent_filter", + default = false + ) + + object EnableGrokGoreFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_grok_gore_filter", + default = false + ) + + object EnableMinVideoDurationFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_min_video_duration_filter", + default = false + ) + + object EnableMaxVideoDurationFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_max_video_duration_filter", + default = false + ) + + object EnableClusterBasedDedupFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_cluster_based_dedup_filter", + default = false + ) + + object EnableCountryFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_country_filter", + default = false + ) + + object EnableRegionFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_region_filter", + default = false + ) + + object EnableHasMultipleMediaFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_has_multiple_media_filter", + default = false + ) + + object EnableClusterBased88DedupFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_cluster_based_88_dedup_filter", + default = false + ) + + object EnableNoClusterFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_no_cluster_filter", + default = false + ) + + object DedupHistoricalEventsTimeWindowParam + extends FSBoundedParam[Long]( + name = "home_mixer_dedup_historical_events_time_window", + default = 43200000L, // 12 * 60 * 60 * 1000 = 12hrs in milliseconds + min = 0L, + max = 604800000L // 7 days + ) + + object MinVideoDurationThresholdParam + extends FSBoundedParam[Long]( + name = "home_mixer_min_video_duration_threshold", + default = 0L, // 0 second + min = 0L, + max = 604800000L // 7 days + ) + + object MaxVideoDurationThresholdParam + extends FSBoundedParam[Long]( + name = "home_mixer_max_video_duration_threshold", + default = 604800000L, // 7 days + min = 0L, + max = 604800000L // 7 days + ) + + object EnableSlopFilterLowSignalUsers + extends FSParam[Boolean]( + name = "home_mixer_enable_slop_filter_low_signal_users", + default = false + ) + + object EnableSlopFilterEligibleUserStateParam + extends FSParam[Boolean]( + name = "home_mixer_enable_slop_filter_eligible_user_state_param", + default = true + ) + + object SlopMaxScore + extends FSBoundedParam[Double]( + name = "home_mixer_slop_max_score", + default = 0.3, + min = 0.0, + max = 4.0 + ) + + object SlopMinFollowers + extends FSBoundedParam[Int]( + name = "home_mixer_slop_min_followers", + default = 100, + min = 0, + max = 10000000 + ) + + object EnableGrokAnnotations + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_grok_annotations", + default = false + ) + + object UserActionsMaxCount + extends FSBoundedParam[Int]( + name = "home_mixer_user_actions_max_count", + default = 522, + min = 0, + max = 10000 + ) + + object EnableTweetRTAMhOnlyParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_tweet_rta_read_from_mh", + default = false + ) + + object EnableTweetRTAMhFallbackParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_tweet_rta_read_fallback_to_mh", + default = false + ) + + object EnableTweetCountryRTAMhOnlyParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_tweet_country_rta_read_from_mh", + default = false + ) + + object EnableTweetCountryRTAMhFallbackParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_tweet_country_rta_read_fallback_to_mh", + default = false + ) + + object EnableUserRTAMhOnlyParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_user_rta_read_from_mh", + default = false + ) + + object EnableUserRTAMhFallbackParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_user_rta_read_fallback_to_mh", + default = false + ) + + object EnableUserAuthorRTAMhOnlyParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_user_author_rta_read_from_mh", + default = false + ) + + object EnableUserAuthorRTAMhFallbackParam + extends FSParam[Boolean]( + name = "home_mixer_feature_hydration_enable_user_author_rta_read_fallback_to_mh", + default = false + ) + + object MaxPostContextPostsPerRequest + extends FSParam[Int]( + name = "home_mixer_feature_hydration_max_post_context_posts", + default = 5 + ) + + object MaxPostContextDuplicatesPerRequest + extends FSParam[Int]( + name = "home_mixer_feature_hydration_max_post_context_duplicates", + default = 2 + ) + + object PhoenixCluster extends Enumeration { + val Prod = Value + val Experiment1 = Value + val Experiment2 = Value + val Experiment3 = Value + val Experiment4 = Value + val Experiment5 = Value + val Experiment6 = Value + val Experiment7 = Value + val Experiment8 = Value + } + + object PhoenixInferenceClusterParam + extends FSEnumParam[PhoenixCluster.type]( + name = "home_mixer_model_phoenix_inference_cluster_id", + default = PhoenixCluster.Prod, + enum = PhoenixCluster + ) + + object PhoenixTimeoutInMsParam + extends FSBoundedParam[Int]( + name = "home_mixer_model_phoenix_timeout_in_ms", + default = 500, + min = 10, + max = 10000 + ) + + object EnablePhoenixScorerParam + extends FSParam[Boolean]( + name = "home_mixer_model_enable_phoenix_scorer", + default = false + ) + + object EnableUserActionsShadowScribeParam + extends FSParam[Boolean]( + name = "home_mixer_enable_user_actions_shadow_scribe", + default = false + ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala index dc5f5513f..433538997 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala @@ -4,10 +4,8 @@ object HomeMixerFlagName { final val ScribeClientEventsFlag = "scribe.client_events" final val ScribeServedCandidatesFlag = "scribe.served_candidates" final val ScribeScoredCandidatesFlag = "scribe.scored_candidates" - final val ScribeServedCommonFeaturesAndCandidateFeaturesFlag = - "scribe.served_common_features_and_candidate_features" + final val ScribeFeaturesFlag = "scribe.features" final val DataRecordMetadataStoreConfigsYmlFlag = "data.record.metadata.store.configs.yml" final val DarkTrafficFilterDeciderKey = "thrift.dark.traffic.filter.decider_key" - final val TargetFetchLatency = "target.fetch.latency" final val TargetScoringLatency = "target.scoring.latency" } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala index 5ea87f5ab..1b5876546 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala @@ -2,26 +2,46 @@ package com.twitter.home_mixer.param object HomeMixerInjectionNames { final val AuthorFeatureRepository = "AuthorFeatureRepository" - final val CandidateFeaturesScribeEventPublisher = "CandidateFeaturesScribeEventPublisher" final val CommonFeaturesScribeEventPublisher = "CommonFeaturesScribeEventPublisher" + final val CommonFeaturesScribeVideoEventPublisher = "CommonFeaturesScribeVideoEventPublisher" final val EarlybirdRepository = "EarlybirdRepository" + final val EarlybirdRealtimCGEndpoint = "EarlybirdRealtimCGEndpoint" final val EngagementsReceivedByAuthorCache = "EngagementsReceivedByAuthorCache" + final val EntityRealGraphClientStore = "EntityRealGraphClientStore" final val GraphTwoHopRepository = "GraphTwoHopRepository" + final val GizmoduckTimelinesCache = "GizmoduckTimelinesCache" final val HomeAuthorFeaturesCacheClient = "HomeAuthorFeaturesCacheClient" final val InterestsThriftServiceClient = "InterestsThriftServiceClient" + final val BatchedStratoClientWithDefaultTimeout = "BatchedStratoClientWithDefaultTimeout" + final val StratoClientWithLongTimeout = "StratoClientWithLongTimeout" final val BatchedStratoClientWithModerateTimeout = "BatchedStratoClientWithModerateTimeout" + final val BatchedStratoClientWithLongTimeout = "BatchedStratoClientWithLongTimeout" final val ManhattanApolloClient = "ManhattanApolloClient" final val ManhattanAthenaClient = "ManhattanAthenaClient" final val ManhattanOmegaClient = "ManhattanOmegaClient" final val ManhattanStarbuckClient = "ManhattanStarbuckClient" - final val MetricCenterUserCountingFeatureRepository = "MetricCenterUserCountingFeatureRepository" - final val MinimumFeaturesScribeEventPublisher = "MinimumFeaturesScribeEventPublisher" - final val RealGraphInNetworkScores = "RealGraphInNetworkScores" + final val MediaClipClusterIdInMemCache = "MediaClipClusterIdInMemCache" + final val ImageClipClusterIdInMemCache = "ImageClipClusterIdInMemCache" + final val MediaCompletionRateInMemCache = "MediaCompletionRateInMemCache" + final val IsColdStartPostInMemCache = "IsColdStartPostInMemCache" + final val MemcacheCandidateFeaturesStore = "MemcacheCandidateFeaturesStore" + final val MemcacheVideoCandidateFeaturesStore = "MemcacheVideoCandidateFeaturesStore" + final val MemcachedImpressionBloomFilterStore = "MemcachedImpressionBloomFilterStore" + final val NaviModelClientHomeRecap = "NaviModelClientHomeRecap" + final val NaviModelClientHomeRecapSecondary = "NaviModelClientHomeRecapSecondary" + final val NaviModelClientHomeRecapRealtime = "NaviModelClientHomeRecapRealtime" + final val NaviModelClientHomeRecapGPU = "NaviModelClientHomeRecapGPU" + final val NaviModelClientHomeRecapVideo = "NaviModelClientHomeRecapVideo" + final val RealGraphInNetworkScoresOnPrem = "RealGraphInNetworkScoresOnPrem" final val RealGraphManhattanEndpoint = "RealGraphFeaturesManhattanEndpoint" + final val RTAManhattanEndpoint = "RTAManhattanEndpoint" + final val RTAManhattanStore = "RTAManhattanStore" final val RealGraphFeatureRepository = "RealGraphFeatureRepository" final val RealTimeInteractionGraphUserVertexCache = "RealTimeInteractionGraphUserVertexCache" final val RealTimeInteractionGraphUserVertexClient = "RealTimeInteractionGraphUserVertexClient" - final val StaleTweetsCache = "StaleTweetsCache" + final val ScoredTweetsCache = "ScoredTweetsCache" + final val ScoredVideoTweetsCache = "ScoredVideoTweetsCache" + final val TesBatchedStratoClient = "TesBatchedStratoClient" final val TimelineAggregateMetadataRepository = "TimelineAggregateMetadataRepository" final val TimelineAggregatePartARepository = "TimelineAggregatePartARepository" final val TimelineAggregatePartBRepository = "TimelineAggregatePartBRepository" @@ -30,12 +50,19 @@ object HomeMixerInjectionNames { final val TopicEngagementCache = "TopicEngagementCache" final val TweetCountryEngagementCache = "TweetCountryEngagementCache" final val TweetEngagementCache = "TweetEngagementCache" - final val TweetypieContentRepository = "TweetypieContentRepository" final val TweetypieStaticEntitiesCache = "TweetypieStaticEntitiesCache" final val TwhinAuthorFollowFeatureCacheClient = "TwhinAuthorFollowFeatureCacheClient" final val TwhinAuthorFollowFeatureRepository = "TwhinAuthorFollowFeatureRepository" final val TwhinUserEngagementFeatureRepository = "TwhinUserEngagementFeatureRepository" + final val TwhinRebuildUserEngagementFeatureRepository = + "TwhinRebuildUserEngagementFeatureRepository" final val TwhinUserFollowFeatureRepository = "TwhinUserFollowFeatureRepository" + final val TwhinTweetEmbeddingsStore = "TwhinTweetEmbeddingsStore" + final val TwhinRebuildTweetEmbeddingsStore = "TwhinRebuildTweetEmbeddingsStore" + final val TwhinVideoEmbeddingsStore = "TwhinVideoEmbeddingsStore" + final val TwhinUserNegativeEmbeddingsStore = "TwhinUserNegativeEmbeddingsStore" + final val TwhinUserPositiveEmbeddingsStore = "TwhinUserPositiveEmbeddingsStore" + final val TwhinRebuildUserPositiveEmbeddingsStore = "TwhinRebuildUserPositiveEmbeddingsStore" final val TwitterListEngagementCache = "TwitterListEngagementCache" final val UserAuthorEngagementCache = "UserAuthorEngagementCache" final val UserEngagementCache = "UserEngagementCache" @@ -43,4 +70,12 @@ object HomeMixerInjectionNames { final val UserLanguagesRepository = "UserLanguagesRepository" final val UserTopicEngagementForNewUserCache = "UserTopicEngagementForNewUserCache" final val UtegSocialProofRepository = "UtegSocialProofRepository" + final val TvRealTimeAggregateClient = "TvRealTimeAggregateClient" + final val TvVideoByUserTweetCache = "TvVideoByUserTweetCache" + final val TvWatchedVideoIdsKeyCacheStore = "TvWatchedVideoIdsKeyCacheStore" + final val MediaClusterId95Store = "MediaClusterId95Store" + final val MediaClusterId88Store = "MediaClusterId88Store" + final val TweetWatchTimeMetadataStore = "TweetWatchTimeMetadataStore" + final val VideoEmbeddingMHStore = "VideoEmbeddingMHStore" + } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.bazel index 5974c186f..18061bba2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.bazel @@ -4,7 +4,5 @@ scala_library( platform = "java8", strict_deps = True, tags = ["bazel-compatible"], - dependencies = [ - "servo/decider", - ], + dependencies = ["servo/decider"], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala index 91f7646d9..9274a8a1c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala @@ -9,44 +9,53 @@ import com.twitter.servo.decider.DeciderKeyEnum * @see [[com.twitter.product_mixer.core.product.ProductParamConfig.enabledDeciderKey]] */ object DeciderKey extends DeciderKeyEnum { - // Products + val EnableForYouProduct = Value("enable_for_you_product") val EnableFollowingProduct = Value("enable_following_product") val EnableScoredTweetsProduct = Value("enable_scored_tweets_product") - val EnableListTweetsProduct = Value("enable_list_tweets_product") - - val EnableListRecommendedUsersProduct = Value("enable_list_recommended_users_product") + val EnableScoredVideoTweetsProduct = Value("enable_scored_video_tweets_product") val EnableSubscribedProduct = Value("enable_subscribed_product") - // Candidate Pipelines - val EnableForYouScoredTweetsCandidatePipeline = - Value("enable_for_you_scored_tweets_candidate_pipeline") + val EnableHeavyRankerScoresProduct = Value("enable_heavy_ranker_scores_product") - val EnableScoredTweetsTweetMixerCandidatePipeline = - Value("enable_scored_tweets_tweet_mixer_candidate_pipeline") + val EnableSimClustersSimilarityFeatureHydration = + Value("enable_simclusters_similarity_feature_hydration") - val EnableScoredTweetsInNetworkCandidatePipeline = - Value("enable_scored_tweets_in_network_candidate_pipeline") + val EnableServedCandidateFeatureKeysKafkaPublishing = + Value("enable_served_candidate_feature_keys_kafka_publishing") - val EnableScoredTweetsUtegCandidatePipeline = - Value("enable_scored_tweets_uteg_candidate_pipeline") + val EnablePublishCommonFeaturesKafka = + Value("enable_publish_common_features_kafka") - val EnableScoredTweetsFrsCandidatePipeline = - Value("enable_scored_tweets_frs_candidate_pipeline") + val LiveSpacesFactor = Value("live_spaces_factor") - val EnableScoredTweetsListsCandidatePipeline = - Value("enable_scored_tweets_lists_candidate_pipeline") + val MtlNormalizationAlpha = Value("mtl_normalization_alpha") - val EnableScoredTweetsPopularVideosCandidatePipeline = - Value("enable_scored_tweets_popular_videos_candidate_pipeline") + val EnableTweetypieContentFeatures = Value("enable_tweetypie_content_features") - val EnableScoredTweetsBackfillCandidatePipeline = - Value("enable_scored_tweets_backfill_candidate_pipeline") + val EnableCommonFeaturesDataRecordCopyDuringPldrConversion = + Value("enable_common_features_data_record_copy_during_pldr_conversion") - val EnableSimClustersSimilarityFeatureHydration = - Value("enable_simclusters_similarity_feature_hydration") + val EnableGetTweetsFromArchiveIndex = Value("enable_get_tweets_from_archive_index") + + val EnableSecondaryNaviRecapCluster = Value("enable_secondary_navi_recap_cluster") + + val EnableGPUNaviRecapClusterTestUsers = Value("enable_gpu_navi_recap_cluster_test_users") + + val NaviGPUClusterRequestBatchSize = Value("navi_gpu_cluster_request_batch_size") + + val EnableVideoSummaryEmbeddingHydration = Value( + "enable_video_summary_embedding_feature_hydration") + + val EnableVideoClipEmbeddingHydration = Value("enable_video_clip_embedding_feature_hydration") + + val EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam = Value( + "enable_scored_video_tweets_user_history_events_query_feature_hydration") + + val EnableVideoClipEmbeddingMediaUnderstandingHydration = Value( + "enable_video_clip_embedding_media_understanding_feature_hydration") } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel index e4fa669d2..ed5d253fe 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel @@ -3,26 +3,19 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/heavy_ranker_scores", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/heavy_ranker_scores/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_video_tweets", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model", "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala index 5db4bd7f6..c6f633642 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala @@ -2,15 +2,15 @@ package com.twitter.home_mixer.product import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct +import com.twitter.home_mixer.model.request.HeavyRankerScoresProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct +import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.product.following.FollowingProductPipelineConfig import com.twitter.home_mixer.product.for_you.ForYouProductPipelineConfig -import com.twitter.home_mixer.product.list_recommended_users.ListRecommendedUsersProductPipelineConfig +import com.twitter.home_mixer.product.heavy_ranker_scores.HeavyRankerScoresProductPipelineConfig import com.twitter.home_mixer.product.scored_tweets.ScoredTweetsProductPipelineConfig -import com.twitter.home_mixer.product.list_tweets.ListTweetsProductPipelineConfig +import com.twitter.home_mixer.product.scored_video_tweets.ScoredVideoTweetsProductPipelineConfig import com.twitter.home_mixer.product.subscribed.SubscribedProductPipelineConfig import com.twitter.inject.Injector import com.twitter.product_mixer.core.product.guice.ProductScope @@ -27,7 +27,6 @@ class HomeProductPipelineRegistryConfig @Inject() ( private val followingProductPipelineConfig = productScope.let(FollowingProduct) { injector.instance[FollowingProductPipelineConfig] } - private val forYouProductPipelineConfig = productScope.let(ForYouProduct) { injector.instance[ForYouProductPipelineConfig] } @@ -36,25 +35,24 @@ class HomeProductPipelineRegistryConfig @Inject() ( injector.instance[ScoredTweetsProductPipelineConfig] } - private val listTweetsProductPipelineConfig = productScope.let(ListTweetsProduct) { - injector.instance[ListTweetsProductPipelineConfig] + private val scoredVideoTweetsProductPipelineConfig = productScope.let(ScoredVideoTweetsProduct) { + injector.instance[ScoredVideoTweetsProductPipelineConfig] } - private val listRecommendedUsersProductPipelineConfig = - productScope.let(ListRecommendedUsersProduct) { - injector.instance[ListRecommendedUsersProductPipelineConfig] - } - private val subscribedProductPipelineConfig = productScope.let(SubscribedProduct) { injector.instance[SubscribedProductPipelineConfig] } + private val heavyRankerScoresProductPipelineConfig = productScope.let(HeavyRankerScoresProduct) { + injector.instance[HeavyRankerScoresProductPipelineConfig] + } + override val productPipelineConfigs = Seq( followingProductPipelineConfig, forYouProductPipelineConfig, scoredTweetsProductPipelineConfig, - listTweetsProductPipelineConfig, - listRecommendedUsersProductPipelineConfig, + scoredVideoTweetsProductPipelineConfig, subscribedProductPipelineConfig, + heavyRankerScoresProductPipelineConfig ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel index 4771fe655..c67ecf0f6 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel @@ -4,13 +4,8 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", - "finagle/finagle-memcached/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", @@ -19,7 +14,6 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", @@ -27,65 +21,34 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline/selector", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/cursor/timelines", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", "src/java/com/twitter/search/common/schema/base", "src/java/com/twitter/search/common/schema/earlybird", - "src/java/com/twitter/search/common/util/lang", "src/java/com/twitter/search/queryparser/query:core-query-nodes", "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", - "src/scala/com/twitter/suggests/controller_data", "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/suggests/controller_data:controller_data-scala", - "src/thrift/com/twitter/timelinemixer:thrift-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "stitch/stitch-gizmoduck", - "stitch/stitch-tweetypie", - "stringcenter/client", - "stringcenter/client/src/main/java", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", ], exports = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala index d4898eb31..59bbc1287 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala @@ -2,15 +2,15 @@ package com.twitter.home_mixer.product.following import com.twitter.adserver.{thriftscala => ads} import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator.NoAdsTierFeature import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate -import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature +import com.twitter.home_mixer.functional_component.gate.ExcludeSyntheticUserGate import com.twitter.home_mixer.param.HomeGlobalParams import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.home_mixer.product.following.param.FollowingParam.EnableAdsCandidatePipelineParam import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFastAds import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder @@ -18,26 +18,21 @@ import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate +import com.twitter.product_mixer.component_library.gate.FeatureGate import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountCandidatesFromPipelines -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.PipelineScopedOrganicItems +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfig +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter -import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.gate.ParamNotGate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} import javax.inject.Inject import javax.inject.Singleton @Singleton class FollowingAdsCandidatePipelineBuilder @Inject() ( - adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, + adsCandidatePipelineConfigBuilder: AdsCandidatePipelineConfigBuilder, adsCandidateSource: AdsProdThriftCandidateSource, advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ FollowingQuery, @@ -46,11 +41,13 @@ class FollowingAdsCandidatePipelineBuilder @Inject() ( private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("FollowingAds") - private val suggestType = st.SuggestType.Promoted + private val EstimatedNumOrganicItems = 100.toShort + + private val servedType = hmt.ServedType.FollowingPromoted private val clientEventInfoBuilder = ClientEventInfoBuilder( - component = InjectionScribeUtil.scribeComponent(suggestType).get, - detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) + component = servedType.originalName, + detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(servedType.originalName))) ) private val contextualTweetRefBuilder = ContextualTweetRefBuilder( @@ -70,29 +67,22 @@ class FollowingAdsCandidatePipelineBuilder @Inject() ( HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) - def build( - organicCandidatePipelines: CandidateScope - ): AdsDependentCandidatePipelineConfig[FollowingQuery] = + def build(): AdsCandidatePipelineConfig[FollowingQuery] = adsCandidatePipelineConfigBuilder.build[FollowingQuery]( adsCandidateSource = adsCandidateSource, identifier = identifier, adsDisplayLocationBuilder = query => if (query.params.getBoolean(EnableFastAds)) ads.DisplayLocation.TimelineHomeReverseChron else ads.DisplayLocation.TimelineHome, - getOrganicItems = PipelineScopedOrganicItems( - pipelines = organicCandidatePipelines, - textFeature = TweetTextFeature, - languageFeature = TweetLanguageFeature - ), - countNumOrganicItems = CountCandidatesFromPipelines(organicCandidatePipelines), - supportedClientParam = Some(EnableAdsCandidatePipelineParam), + estimateNumOrganicItems = _ => EstimatedNumOrganicItems, gates = Seq( ParamNotGate( name = "AdsDisableInjectionBasedOnUserRole", param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam ), ExcludeSoftUserGate, - NonEmptyCandidatesGate(organicCandidatePipelines) + ExcludeSyntheticUserGate, + FeatureGate.fromNegatedFeature(NoAdsTierFeature) ), filters = Seq(ValidAdImpressionIdFilter), postFilterFeatureHydration = Seq( @@ -103,6 +93,6 @@ class FollowingAdsCandidatePipelineBuilder @Inject() ( ), decorator = Some(decorator), alerts = alerts, - urtRequest = Some(true), + urtRequest = Some(true) ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsDependentCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsDependentCandidatePipelineBuilder.scala new file mode 100644 index 000000000..d4a043682 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsDependentCandidatePipelineBuilder.scala @@ -0,0 +1,111 @@ +package com.twitter.home_mixer.product.following + +import com.twitter.adserver.{thriftscala => ads} +import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator.NoAdsTierFeature +import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate +import com.twitter.home_mixer.functional_component.gate.ExcludeSyntheticUserGate +import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature +import com.twitter.home_mixer.param.HomeGlobalParams +import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam +import com.twitter.home_mixer.product.following.model.FollowingQuery +import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFastAds +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator +import com.twitter.product_mixer.component_library.gate.FeatureGate +import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate +import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountCandidatesFromPipelines +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.PipelineScopedOrganicItems +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.gate.ParamNotGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel +import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class FollowingAdsDependentCandidatePipelineBuilder @Inject() ( + adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, + adsCandidateSource: AdsProdThriftCandidateSource, + advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ + FollowingQuery, + AdsCandidate + ]) { + + private val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("FollowingAdsDependent") + + private val servedType = hmt.ServedType.FollowingPromoted + + private val clientEventInfoBuilder = ClientEventInfoBuilder( + component = servedType.originalName, + detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(servedType.originalName))) + ) + + private val contextualTweetRefBuilder = ContextualTweetRefBuilder( + TweetHydrationContext( + safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), + outerTweetContext = None + )) + + private val decorator = UrtItemCandidateDecorator( + AdsCandidateUrtItemBuilder( + tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), + contextualTweetRefBuilder = Some(contextualTweetRefBuilder) + )) + + private val alerts = Seq( + HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), + HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() + ) + + def build( + organicCandidatePipelines: CandidateScope + ): AdsDependentCandidatePipelineConfig[FollowingQuery] = + adsCandidatePipelineConfigBuilder.build[FollowingQuery]( + adsCandidateSource = adsCandidateSource, + identifier = identifier, + adsDisplayLocationBuilder = query => + if (query.params.getBoolean(EnableFastAds)) ads.DisplayLocation.TimelineHomeReverseChron + else ads.DisplayLocation.TimelineHome, + getOrganicItems = PipelineScopedOrganicItems( + pipelines = organicCandidatePipelines, + textFeature = TweetTextFeature, + languageFeature = TweetLanguageFeature + ), + countNumOrganicItems = CountCandidatesFromPipelines(organicCandidatePipelines), + gates = Seq( + ParamNotGate( + name = "AdsDisableInjectionBasedOnUserRole", + param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam + ), + ExcludeSoftUserGate, + ExcludeSyntheticUserGate, + FeatureGate.fromNegatedFeature(NoAdsTierFeature), + NonEmptyCandidatesGate(organicCandidatePipelines) + ), + filters = Seq(ValidAdImpressionIdFilter), + postFilterFeatureHydration = Seq( + ParamGatedCandidateFeatureHydrator( + EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, + advertiserBrandSafetySettingsFeatureHydrator + ) + ), + decorator = Some(decorator), + alerts = alerts, + urtRequest = Some(true), + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingDependentAdsMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingDependentAdsMixerPipelineConfig.scala new file mode 100644 index 000000000..5d1d3edce --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingDependentAdsMixerPipelineConfig.scala @@ -0,0 +1,315 @@ +package com.twitter.home_mixer.product.following + +import com.twitter.clientapp.{thriftscala => ca} +import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas +import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder +import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig +import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig +import com.twitter.home_mixer.candidate_pipeline.VerifiedPromptCandidatePipelineConfig +import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator._ +import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails +import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration +import com.twitter.home_mixer.functional_component.side_effect._ +import com.twitter.home_mixer.model.ClearCacheIncludeInstruction +import com.twitter.home_mixer.model.NavigationIncludeInstruction +import com.twitter.home_mixer.param.HomeGlobalParams.EnableSSPAdsBrandSafetySettingsFeatureHydratorParam +import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag +import com.twitter.home_mixer.product.following.model.FollowingQuery +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.product.following.param.FollowingParam.ClearCache +import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFlipInjectionModuleCandidatePipelineParam +import com.twitter.home_mixer.product.following.param.FollowingParam.EnablePostContextFeatureHydratorParam +import com.twitter.home_mixer.product.following.param.FollowingParam.Navigation +import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam +import com.twitter.home_mixer.product.following.param.FollowingParam.StaticParamValueFive +import com.twitter.home_mixer.product.following.param.FollowingParam.StaticParamValueZero +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.inject.annotations.Flag +import com.twitter.logpipeline.client.common.EventPublisher +import com.twitter.product_mixer.component_library.feature_hydrator.query.ads.SSPAdsBrandSafetySettingsFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.location.UserLocationQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptDependentCandidatePipelineConfigBuilder +import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.selector.FlipPromptDynamicInsertionPosition +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig +import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.NavigationInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedCursorIdSelector.TweetIdSelector +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.earlybird.EarlybirdGapIncludeInstruction +import com.twitter.product_mixer.component_library.selector.DropMaxCandidates +import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates +import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults +import com.twitter.product_mixer.component_library.selector.InsertAppendResults +import com.twitter.product_mixer.component_library.selector.InsertDynamicPositionResults +import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults +import com.twitter.product_mixer.component_library.selector.SelectConditionally +import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates +import com.twitter.product_mixer.component_library.selector.ads.AdsInjector +import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults +import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline +import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines +import com.twitter.product_mixer.core.functional_component.configapi.StaticParam +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller +import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller +import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline +import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig +import com.twitter.product_mixer.core.pipeline.FailOpenPolicy +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig +import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelines.render.{thriftscala => urt} +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +class FollowingDependentAdsMixerPipelineConfig @Inject() ( + followingEarlybirdCandidatePipelineConfig: FollowingEarlybirdCandidatePipelineConfig, + conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ + FollowingQuery + ], + followingAdsDependentCandidatePipelineBuilder: FollowingAdsDependentCandidatePipelineBuilder, + followingWhoToFollowCandidatePipelineConfigBuilder: FollowingWhoToFollowCandidatePipelineConfigBuilder, + flipPromptDependentCandidatePipelineConfigBuilder: FlipPromptDependentCandidatePipelineConfigBuilder, + editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, + newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[FollowingQuery], + verifiedPromptCandidatePipelineConfig: VerifiedPromptCandidatePipelineConfig, + dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, + gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, + persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, + rateLimitQueryFeatureHydrator: RateLimitQueryFeatureHydrator, + requestQueryFeatureHydrator: RequestQueryFeatureHydrator[FollowingQuery], + sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, + memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, + lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, + userLocationQueryFeatureHydrator: UserLocationQueryFeatureHydrator, + userSubscriptionQueryFeatureHydrator: UserSubscriptionQueryFeatureHydrator, + sspAdsBrandSafetySettingsFeatureHydrator: SSPAdsBrandSafetySettingsFeatureHydrator, + adsInjector: AdsInjector, + updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[FollowingQuery, Timeline], + publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, + updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, + truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, + homeTimelineServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, + clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], + externalStrings: HomeMixerExternalStrings, + @ProductScoped stringCenterProvider: Provider[StringCenter], + urtTransportMarshaller: UrtTransportMarshaller, + @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) + extends MixerPipelineConfig[FollowingQuery, Timeline, urt.TimelineResponse] { + + override val identifier: MixerPipelineIdentifier = + MixerPipelineIdentifier("FollowingDependentAds") + + private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep + private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep + + override val fetchQueryFeatures: Seq[QueryFeatureHydrator[FollowingQuery]] = Seq( + requestQueryFeatureHydrator, + sgsFollowedUsersQueryFeatureHydrator, + rateLimitQueryFeatureHydrator, + gizmoduckUserQueryFeatureHydrator, + userSubscriptionQueryFeatureHydrator, + ParamGatedQueryFeatureHydrator( + EnableSSPAdsBrandSafetySettingsFeatureHydratorParam, + sspAdsBrandSafetySettingsFeatureHydrator + ), + AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), + AsyncQueryFeatureHydrator(dependentCandidatesStep, persistenceStoreQueryFeatureHydrator), + AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), + AsyncQueryFeatureHydrator(dependentCandidatesStep, userLocationQueryFeatureHydrator), + AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator), + ) + + private val earlybirdCandidatePipelineScope = + SpecificPipeline(followingEarlybirdCandidatePipelineConfig.identifier) + + private val conversationServiceCandidatePipelineConfig = + conversationServiceCandidatePipelineConfigBuilder.build( + earlybirdCandidatePipelineScope, + hmt.ServedType.FollowingInNetwork, + EnablePostContextFeatureHydratorParam + ) + + private val followingAdsCandidatePipelineConfig = + followingAdsDependentCandidatePipelineBuilder.build(earlybirdCandidatePipelineScope) + + private val followingWhoToFollowCandidatePipelineConfig = + followingWhoToFollowCandidatePipelineConfigBuilder.build(earlybirdCandidatePipelineScope) + + private val flipPromptCandidatePipelineConfig = + flipPromptDependentCandidatePipelineConfigBuilder.build[FollowingQuery]( + supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) + ) + + override val candidatePipelines: Seq[CandidatePipelineConfig[FollowingQuery, _, _, _]] = + Seq(followingEarlybirdCandidatePipelineConfig) + + override val dependentCandidatePipelines: Seq[ + DependentCandidatePipelineConfig[FollowingQuery, _, _, _] + ] = Seq( + conversationServiceCandidatePipelineConfig, + followingAdsCandidatePipelineConfig, + followingWhoToFollowCandidatePipelineConfig, + flipPromptCandidatePipelineConfig, + editedTweetsCandidatePipelineConfig, + newTweetsPillCandidatePipelineConfig, + verifiedPromptCandidatePipelineConfig + ) + + override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( + followingAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + followingWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + ) + + override val resultSelectors: Seq[Selector[FollowingQuery]] = Seq( + UpdateSortCandidates( + ordering = CandidatesUtil.reverseChronTweetsOrdering, + candidatePipeline = conversationServiceCandidatePipelineConfig.identifier + ), + DropMaxCandidates( + candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, + maxSelectionsParam = MaxNumberReplaceInstructionsParam + ), + DropMaxCandidates( + candidatePipeline = conversationServiceCandidatePipelineConfig.identifier, + maxSelectionsParam = ServerMaxResultsParam + ), + DropModuleTooFewModuleItemResults( + candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, + minModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MinCandidatesSize) + ), + DropMaxModuleItemCandidates( + candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, + maxModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MaxCandidatesSize) + ), + InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), + InsertFixedPositionResults( + candidatePipeline = verifiedPromptCandidatePipelineConfig.identifier, + positionParam = StaticParamValueZero + ), + InsertDynamicPositionResults( + candidatePipeline = flipPromptCandidatePipelineConfig.identifier, + dynamicInsertionPosition = FlipPromptDynamicInsertionPosition(StaticParamValueZero) + ), + InsertFixedPositionResults( + candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, + positionParam = StaticParamValueFive + ), + InsertAdResults( + surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, + adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), + adsCandidatePipeline = followingAdsCandidatePipelineConfig.identifier + ), + // This selector must come after the tweets are inserted into the results + UpdateNewTweetsPillDecoration( + pipelineScope = SpecificPipelines( + conversationServiceCandidatePipelineConfig.identifier, + newTweetsPillCandidatePipelineConfig.identifier + ), + stringCenter = stringCenterProvider.get(), + seeNewTweetsString = externalStrings.seeNewTweetsString, + tweetedString = externalStrings.tweetedString + ), + InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), + SelectConditionally( + selector = + InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), + includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) + ), + UpdateHomeClientEventDetails( + candidatePipelines = Set(conversationServiceCandidatePipelineConfig.identifier) + ) + ) + + private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( + enableScribeClientEvents = enableScribeClientEvents, + logPipelinePublisher = clientEventsScribeEventPublisher, + injectedTweetsCandidatePipelineIdentifiers = + Seq(conversationServiceCandidatePipelineConfig.identifier), + adsCandidatePipelineIdentifier = Some(followingAdsCandidatePipelineConfig.identifier), + whoToFollowCandidatePipelineIdentifier = + Some(followingWhoToFollowCandidatePipelineConfig.identifier), + ) + + override val resultSideEffects: Seq[PipelineResultSideEffect[FollowingQuery, Timeline]] = Seq( + homeScribeClientEventSideEffect, + homeTimelineServedCandidatesSideEffect, + publishClientSentImpressionsEventBusSideEffect, + truncateTimelinesPersistenceStoreSideEffect, + updateLastNonPollingTimeSideEffect, + updateTimelinesPersistenceStoreSideEffect, + ) + + override val domainMarshaller: DomainMarshaller[FollowingQuery, Timeline] = { + val instructionBuilders = Seq( + ClearCacheInstructionBuilder( + ClearCacheIncludeInstruction( + ClearCache.PtrEnableParam, + ClearCache.ColdStartEnableParam, + ClearCache.WarmStartEnableParam, + ClearCache.ManualRefreshEnableParam, + ClearCache.NavigateEnableParam, + ClearCache.MinEntriesParam + )), + ReplaceEntryInstructionBuilder(ReplaceAllEntries), + AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), + ShowAlertInstructionBuilder(), + ShowCoverInstructionBuilder(), + NavigationInstructionBuilder( + NavigationIncludeInstruction( + Navigation.PtrEnableParam, + Navigation.ColdStartEnableParam, + Navigation.WarmStartEnableParam, + Navigation.ManualRefreshEnableParam, + Navigation.NavigateEnableParam + )) + ) + + val topCursorBuilder = OrderedTopCursorBuilder(TweetIdSelector) + val bottomCursorBuilder = + OrderedBottomCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction.inverse()) + val gapCursorBuilder = OrderedGapCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction) + + val scribeConfigBuilder = + StaticTimelineScribeConfigBuilder(TimelineScribeConfig(Some("following"), None, None)) + val metadataBuilder = UrtMetadataBuilder(scribeConfigBuilder = Some(scribeConfigBuilder)) + + UrtDomainMarshaller( + instructionBuilders = instructionBuilders, + metadataBuilder = Some(metadataBuilder), + cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder, gapCursorBuilder) + ) + } + + override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = + urtTransportMarshaller +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala index addb298c2..83dfeeb97 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala @@ -1,8 +1,9 @@ package com.twitter.home_mixer.product.following import com.twitter.home_mixer.candidate_pipeline.FollowingEarlybirdResponseFeatureTransformer -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource +import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.product.following.model.FollowingQuery +import com.twitter.product_mixer.component_library.candidate_source.earlybird.EarlybirdTweetCandidateSource import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate @@ -19,7 +20,7 @@ import javax.inject.Singleton @Singleton class FollowingEarlybirdCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, + earlybirdTweetCandidateSource: EarlybirdTweetCandidateSource, followingEarlybirdQueryTransformer: FollowingEarlybirdQueryTransformer) extends CandidatePipelineConfig[ FollowingQuery, @@ -32,9 +33,10 @@ class FollowingEarlybirdCandidatePipelineConfig @Inject() ( CandidatePipelineIdentifier("FollowingEarlybird") override val candidateSource: BaseCandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] = - earlybirdCandidateSource + earlybirdTweetCandidateSource override val gates: Seq[Gate[FollowingQuery]] = Seq( + RateLimitGate, NonEmptySeqFeatureGate(SGSFollowedUsersFeature) ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala index c388cfa92..127ee5b87 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala @@ -2,7 +2,6 @@ package com.twitter.home_mixer.product.following import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature @@ -27,8 +26,6 @@ case class FollowingEarlybirdQueryTransformer @Inject() (clientId: ClientId) override def transform(query: FollowingQuery): t.EarlybirdRequest = { val followedUserIds = query.features.map(_.get(SGSFollowedUsersFeature)).getOrElse(Seq.empty).toSet - val realGraphInNetworkFollowedUserIds = - query.features.map(_.get(RealGraphInNetworkScoresFeature)).getOrElse(Map.empty).keySet val userId = query.getRequiredUserId val combinedUserIds = userId +: followedUserIds.toSeq diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala index efda03595..96f240ffa 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala @@ -5,36 +5,45 @@ import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig -import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator +import com.twitter.home_mixer.candidate_pipeline.VerifiedPromptCandidatePipelineConfig import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.feature_hydrator._ import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration import com.twitter.home_mixer.functional_component.side_effect._ -import com.twitter.home_mixer.model.GapIncludeInstruction -import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter +import com.twitter.home_mixer.model.ClearCacheIncludeInstruction +import com.twitter.home_mixer.model.NavigationIncludeInstruction +import com.twitter.home_mixer.param.HomeGlobalParams.EnableSSPAdsBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.product.following.param.FollowingParam.ClearCache import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFlipInjectionModuleCandidatePipelineParam -import com.twitter.home_mixer.product.following.param.FollowingParam.FlipInlineInjectionModulePosition +import com.twitter.home_mixer.product.following.param.FollowingParam.EnablePostContextFeatureHydratorParam +import com.twitter.home_mixer.product.following.param.FollowingParam.Navigation import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowPositionParam +import com.twitter.home_mixer.product.following.param.FollowingParam.StaticParamValueFive +import com.twitter.home_mixer.product.following.param.FollowingParam.StaticParamValueZero import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher +import com.twitter.product_mixer.component_library.feature_hydrator.query.ads.SSPAdsBrandSafetySettingsFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate +import com.twitter.product_mixer.component_library.feature_hydrator.query.location.UserLocationQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptDependentCandidatePipelineConfigBuilder +import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.selector.FlipPromptDynamicInsertionPosition import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.NavigationInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedCursorIdSelector.TweetIdSelector import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries @@ -43,10 +52,12 @@ import com.twitter.product_mixer.component_library.premarshaller.urt.builder.Sho import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.earlybird.EarlybirdGapIncludeInstruction import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults import com.twitter.product_mixer.component_library.selector.InsertAppendResults +import com.twitter.product_mixer.component_library.selector.InsertDynamicPositionResults import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults import com.twitter.product_mixer.component_library.selector.SelectConditionally import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates @@ -61,13 +72,10 @@ import com.twitter.product_mixer.core.functional_component.marshaller.response.u import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig @@ -85,31 +93,26 @@ class FollowingMixerPipelineConfig @Inject() ( conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ FollowingQuery ], - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, followingAdsCandidatePipelineBuilder: FollowingAdsCandidatePipelineBuilder, followingWhoToFollowCandidatePipelineConfigBuilder: FollowingWhoToFollowCandidatePipelineConfigBuilder, flipPromptDependentCandidatePipelineConfigBuilder: FlipPromptDependentCandidatePipelineConfigBuilder, editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[FollowingQuery], + verifiedPromptCandidatePipelineConfig: VerifiedPromptCandidatePipelineConfig, dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, - realGraphInNetworkSourceQueryHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, + rateLimitQueryFeatureHydrator: RateLimitQueryFeatureHydrator, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[FollowingQuery], sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ - FollowingQuery - ], - manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ - FollowingQuery - ], memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, + userLocationQueryFeatureHydrator: UserLocationQueryFeatureHydrator, + userSubscriptionQueryFeatureHydrator: UserSubscriptionQueryFeatureHydrator, + sspAdsBrandSafetySettingsFeatureHydrator: SSPAdsBrandSafetySettingsFeatureHydrator, adsInjector: AdsInjector, updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[FollowingQuery, Timeline], publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, homeTimelineServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, @@ -128,17 +131,18 @@ class FollowingMixerPipelineConfig @Inject() ( override val fetchQueryFeatures: Seq[QueryFeatureHydrator[FollowingQuery]] = Seq( requestQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator, - realGraphInNetworkSourceQueryHydrator, + rateLimitQueryFeatureHydrator, + gizmoduckUserQueryFeatureHydrator, + userSubscriptionQueryFeatureHydrator, + ParamGatedQueryFeatureHydrator( + EnableSSPAdsBrandSafetySettingsFeatureHydratorParam, + sspAdsBrandSafetySettingsFeatureHydrator + ), AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, persistenceStoreQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), - AsyncParamGatedQueryFeatureHydrator( - EnableImpressionBloomFilter, - resultSelectorsStep, - impressionBloomFilterQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) + AsyncQueryFeatureHydrator(dependentCandidatesStep, userLocationQueryFeatureHydrator), + AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator), ) private val earlybirdCandidatePipelineScope = @@ -146,12 +150,12 @@ class FollowingMixerPipelineConfig @Inject() ( private val conversationServiceCandidatePipelineConfig = conversationServiceCandidatePipelineConfigBuilder.build( - Seq(NonEmptyCandidatesGate(earlybirdCandidatePipelineScope)), - HomeConversationServiceCandidateDecorator(homeFeedbackActionInfoBuilder) + earlybirdCandidatePipelineScope, + hmt.ServedType.FollowingInNetwork, + EnablePostContextFeatureHydratorParam ) - private val followingAdsCandidatePipelineConfig = - followingAdsCandidatePipelineBuilder.build(earlybirdCandidatePipelineScope) + private val followingAdsCandidatePipelineConfig = followingAdsCandidatePipelineBuilder.build() private val followingWhoToFollowCandidatePipelineConfig = followingWhoToFollowCandidatePipelineConfigBuilder.build(earlybirdCandidatePipelineScope) @@ -161,18 +165,20 @@ class FollowingMixerPipelineConfig @Inject() ( supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) ) - override val candidatePipelines: Seq[CandidatePipelineConfig[FollowingQuery, _, _, _]] = - Seq(followingEarlybirdCandidatePipelineConfig) + override val candidatePipelines: Seq[CandidatePipelineConfig[FollowingQuery, _, _, _]] = Seq( + followingEarlybirdCandidatePipelineConfig, + followingAdsCandidatePipelineConfig + ) override val dependentCandidatePipelines: Seq[ DependentCandidatePipelineConfig[FollowingQuery, _, _, _] ] = Seq( conversationServiceCandidatePipelineConfig, - followingAdsCandidatePipelineConfig, followingWhoToFollowCandidatePipelineConfig, flipPromptCandidatePipelineConfig, editedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig + newTweetsPillCandidatePipelineConfig, + verifiedPromptCandidatePipelineConfig ) override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( @@ -206,12 +212,16 @@ class FollowingMixerPipelineConfig @Inject() ( ), InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), InsertFixedPositionResults( - candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, - positionParam = WhoToFollowPositionParam + candidatePipeline = verifiedPromptCandidatePipelineConfig.identifier, + positionParam = StaticParamValueZero ), - InsertFixedPositionResults( + InsertDynamicPositionResults( candidatePipeline = flipPromptCandidatePipelineConfig.identifier, - positionParam = FlipInlineInjectionModulePosition + dynamicInsertionPosition = FlipPromptDynamicInsertionPosition(StaticParamValueZero) + ), + InsertFixedPositionResults( + candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, + positionParam = StaticParamValueFive ), InsertAdResults( surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, @@ -236,7 +246,7 @@ class FollowingMixerPipelineConfig @Inject() ( ), UpdateHomeClientEventDetails( candidatePipelines = Set(conversationServiceCandidatePipelineConfig.identifier) - ), + ) ) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( @@ -253,8 +263,6 @@ class FollowingMixerPipelineConfig @Inject() ( homeScribeClientEventSideEffect, homeTimelineServedCandidatesSideEffect, publishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect, truncateTimelinesPersistenceStoreSideEffect, updateLastNonPollingTimeSideEffect, updateTimelinesPersistenceStoreSideEffect, @@ -262,30 +270,37 @@ class FollowingMixerPipelineConfig @Inject() ( override val domainMarshaller: DomainMarshaller[FollowingQuery, Timeline] = { val instructionBuilders = Seq( + ClearCacheInstructionBuilder( + ClearCacheIncludeInstruction( + ClearCache.PtrEnableParam, + ClearCache.ColdStartEnableParam, + ClearCache.WarmStartEnableParam, + ClearCache.ManualRefreshEnableParam, + ClearCache.NavigateEnableParam, + ClearCache.MinEntriesParam + )), ReplaceEntryInstructionBuilder(ReplaceAllEntries), AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), ShowAlertInstructionBuilder(), ShowCoverInstructionBuilder(), + NavigationInstructionBuilder( + NavigationIncludeInstruction( + Navigation.PtrEnableParam, + Navigation.ColdStartEnableParam, + Navigation.WarmStartEnableParam, + Navigation.ManualRefreshEnableParam, + Navigation.NavigateEnableParam + )) ) - val idSelector: PartialFunction[UniversalNoun[_], Long] = { - // exclude ads while determining tweet cursor values - case item: TweetItem if item.promotedMetadata.isEmpty => item.id - case module: TimelineModule - if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => - module.items.last.item match { case item: TweetItem => item.id } - } - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) + val topCursorBuilder = OrderedTopCursorBuilder(TweetIdSelector) val bottomCursorBuilder = - OrderedBottomCursorBuilder(idSelector, GapIncludeInstruction.inverse()) - val gapCursorBuilder = OrderedGapCursorBuilder(idSelector, GapIncludeInstruction) + OrderedBottomCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction.inverse()) + val gapCursorBuilder = OrderedGapCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction) - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig(page = Some("following"), section = None, entityToken = None))) - ) + val scribeConfigBuilder = + StaticTimelineScribeConfigBuilder(TimelineScribeConfig(Some("following"), None, None)) + val metadataBuilder = UrtMetadataBuilder(scribeConfigBuilder = Some(scribeConfigBuilder)) UrtDomainMarshaller( instructionBuilders = instructionBuilders, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.scala index 28c7cd6a0..1cd63939b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.scala @@ -1,17 +1,18 @@ package com.twitter.home_mixer.product.following import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.FollowingProductContext import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.product.following.model.FollowingQuery +import com.twitter.home_mixer.product.following.param.FollowingParam.EnableDependentAdsParam import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam import com.twitter.home_mixer.product.following.param.FollowingParamConfig import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer +import com.twitter.product_mixer.component_library.premarshaller.cursor.timelines.ChronologicalCursorUnmarshaller import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy import com.twitter.product_mixer.core.functional_component.common.alert.Alert import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert @@ -44,6 +45,7 @@ import javax.inject.Singleton @Singleton class FollowingProductPipelineConfig @Inject() ( + followingDependentAdsMixerPipelineConfig: FollowingDependentAdsMixerPipelineConfig, followingMixerPipelineConfig: FollowingMixerPipelineConfig, followingParamConfig: FollowingParamConfig) extends ProductPipelineConfig[HomeMixerRequest, FollowingQuery, urt.TimelineResponse] { @@ -97,11 +99,12 @@ class FollowingProductPipelineConfig @Inject() ( ) } - override val pipelines: Seq[PipelineConfig] = Seq(followingMixerPipelineConfig) + override val pipelines: Seq[PipelineConfig] = + Seq(followingMixerPipelineConfig, followingDependentAdsMixerPipelineConfig) - override def pipelineSelector( - query: FollowingQuery - ): ComponentIdentifier = followingMixerPipelineConfig.identifier + override def pipelineSelector(query: FollowingQuery): ComponentIdentifier = + if (query.params(EnableDependentAdsParam)) followingDependentAdsMixerPipelineConfig.identifier + else followingMixerPipelineConfig.identifier override val alerts: Seq[Alert] = Seq( SuccessRateAlert( @@ -112,8 +115,8 @@ class FollowingProductPipelineConfig @Inject() ( LatencyAlert( notificationGroup = DefaultNotificationGroup, percentile = P99, - warnPredicate = TriggerIfLatencyAbove(1100.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(1200.millis, 15, 30) + warnPredicate = TriggerIfLatencyAbove(1200.millis, 15, 30), + criticalPredicate = TriggerIfLatencyAbove(1500.millis, 15, 30) ), ThroughputAlert( notificationGroup = DefaultNotificationGroup, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala index cd5b1795d..02944c556 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala @@ -4,16 +4,17 @@ import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWho import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.param.FollowingParam.EnableWhoToFollowCandidatePipelineParam import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowDisplayLocationParam import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowDisplayTypeIdParam import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowMinInjectionIntervalParam +import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowUserDisplayTypeIdParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.ParamWhoToFollowUserDisplayTypeBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmDependentCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmDependentCandidatePipelineConfigBuilder @@ -37,7 +38,6 @@ class FollowingWhoToFollowCandidatePipelineConfigBuilder @Inject() ( val gates: Seq[BaseGate[PipelineQuery]] = Seq( TimelinesPersistenceStoreLastInjectionGate( WhoToFollowMinInjectionIntervalParam, - PersistenceEntriesFeature, EntityIdType.WhoToFollow ), DismissFatigueGate(SuggestType.WhoToFollow, DismissInfoFeature), @@ -52,6 +52,8 @@ class FollowingWhoToFollowCandidatePipelineConfigBuilder @Inject() ( moduleDisplayTypeBuilder = ParamWhoToFollowModuleDisplayTypeBuilder(WhoToFollowDisplayTypeIdParam), feedbackActionInfoBuilder = Some(homeWhoToFollowFeedbackActionInfoBuilder), + userDisplayTypeBuilder = + ParamWhoToFollowUserDisplayTypeBuilder(WhoToFollowUserDisplayTypeIdParam), displayLocationParam = StaticParam(WhoToFollowDisplayLocationParam.default), excludedUserIdsFeature = Some(WhoToFollowExcludedUserIdsFeature), profileUserIdFeature = None diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.bazel index 70d5e5af9..6f63dcd09 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.bazel @@ -6,13 +6,8 @@ scala_library( dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "stringcenter/client", - "stringcenter/client/src/main/java", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala index 9c6faafa7..bbcd8a354 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala @@ -15,13 +15,26 @@ class HomeMixerExternalStrings @Inject() ( externalStringRegistryProvider.get().createProdString("Tweeted") val muteUserString = externalStringRegistryProvider.get().createProdString("Feedback.muteUser") - val blockUserString = externalStringRegistryProvider.get().createProdString("Feedback.blockUser") + val muteUserConfirmationString = + externalStringRegistryProvider.get().createProdString("Feedback.muteUserConfirmation") + val blockUserString = + externalStringRegistryProvider.get().createProdString("Feedback.blockUser") + val blockUserConfirmationString = + externalStringRegistryProvider.get().createProdString("Feedback.blockUserConfirmation") val unfollowUserString = externalStringRegistryProvider.get().createProdString("Feedback.unfollowUser") val unfollowUserConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.unfollowUserConfirmation") val reportTweetString = externalStringRegistryProvider.get().createProdString("Feedback.reportTweet") + val genericString = + externalStringRegistryProvider.get().createProdString("Feedback.generic") + val genericConfirmationString = + externalStringRegistryProvider.get().createProdString("Feedback.genericConfirmation") + val relevantString = + externalStringRegistryProvider.get().createProdString("Feedback.relevant") + val relevantConfirmationString = + externalStringRegistryProvider.get().createProdString("Feedback.relevantConfirmation") val dontLikeString = externalStringRegistryProvider.get().createProdString("Feedback.dontLike") val dontLikeConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.dontLikeConfirmation") @@ -37,6 +50,39 @@ class HomeMixerExternalStrings @Inject() ( externalStringRegistryProvider.get().createProdString("Feedback.notRelevant") val notRelevantConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.notRelevantConfirmation") + val hatefulString = + externalStringRegistryProvider.get().createProdString("Feedback.hateful") + val hatefulConfirmationString = + externalStringRegistryProvider.get().createProdString("Feedback.hatefulConfirmation") + val boringString = + externalStringRegistryProvider.get().createProdString("Feedback.boring") + val boringConfirmationString = + externalStringRegistryProvider.get().createProdString("Feedback.boringConfirmation") + val confusingString = + externalStringRegistryProvider.get().createProdString("Feedback.confusing") + val confusingConfirmationString = + externalStringRegistryProvider.get().createProdString("Feedback.confusingConfirmation") + val clickbaitString = + externalStringRegistryProvider.get().createProdString("Feedback.clickbait") + val clickbaitConfirmationString = + externalStringRegistryProvider.get().createProdString("Feedback.clickbaitConfirmation") + val ragebaitString = + externalStringRegistryProvider.get().createProdString("Feedback.ragebait") + val ragebaitConfirmationString = + externalStringRegistryProvider.get().createProdString("Feedback.ragebaitConfirmation") + val regretString = + externalStringRegistryProvider.get().createProdString("Feedback.regret") + val regretConfirmationString = + externalStringRegistryProvider.get().createProdString("Feedback.regretConfirmation") + val neutralString = + externalStringRegistryProvider.get().createProdString("Feedback.neutral") + val neutralConfirmationString = + externalStringRegistryProvider.get().createProdString("Feedback.neutralConfirmation") + + val seeMoreString = + externalStringRegistryProvider.get().createProdString("PagedCarouselModule.showMoreText") + val seeLessString = + externalStringRegistryProvider.get().createProdString("PagedCarouselModule.showLessText") val socialContextOneUserLikedString = externalStringRegistryProvider.get().createProdString("SocialContext.oneUserLiked") @@ -64,24 +110,48 @@ class HomeMixerExternalStrings @Inject() ( val socialContextReceivedReply = externalStringRegistryProvider.get().createProdString("SocialContext.receivedReply") - val socialContextPopularVideoString = - externalStringRegistryProvider.get().createProdString("SocialContext.popularVideo") - val socialContextPopularInYourAreaString = externalStringRegistryProvider.get().createProdString("PopgeoTweet.socialProof") - val listToFollowModuleHeaderString = - externalStringRegistryProvider.get().createProdString("ListToFollowModule.header") - val listToFollowModuleFooterString = - externalStringRegistryProvider.get().createProdString("ListToFollowModule.footer") - val pinnedListsModuleHeaderString = - externalStringRegistryProvider.get().createProdString("PinnedListModule.header") - val pinnedListsModuleEmptyStateMessageString = - externalStringRegistryProvider.get().createProdString("PinnedListModule.emptyStateMessage") - val ownedSubscribedListsModuleHeaderString = externalStringRegistryProvider.get().createProdString("OwnedSubscribedListModule.header") - val ownedSubscribedListsModuleEmptyStateMessageString = + + val CommunityToJoinHeaderString = + externalStringRegistryProvider.get().createProdString("CommunityToJoinModule.header") + val CommunityToJoinFooterString = + externalStringRegistryProvider.get().createProdString("CommunityToJoinModule.footer") + + val RecommendedJobHeaderString = + externalStringRegistryProvider.get().createProdString("RecommendedJobModule.header") + val RecommendedJobFooterString = + externalStringRegistryProvider.get().createProdString("RecommendedJobModule.footer") + + val RecommendedRecruitingOrganizationHeaderString = externalStringRegistryProvider - .get().createProdString("OwnedSubscribedListModule.emptyStateMessage") + .get().createProdString("RecommendedRecruitingOrganizationModule.header") + val RecommendedRecruitingOrganizationFooterString = + externalStringRegistryProvider + .get().createProdString("RecommendedRecruitingOrganizationModule.footer") + + val BookmarksHeaderString = + externalStringRegistryProvider.get().createProdString("RecentBookmarks.header") + + val PinnedTweetsHeaderString = + externalStringRegistryProvider.get().createProdString("PinnedTweetsModule.header") + val BroadcastedPinnedTweetSocialContextString = + externalStringRegistryProvider.get().createProdString("BroadcastedPinnedTweet.context") + val VideoCarouselHeaderString = + externalStringRegistryProvider.get().createProdString("VideoCarouselModule.header") + val VideoCarouselFooterString = + externalStringRegistryProvider.get().createProdString("VideoCarouselModule.footer") + + val TrendingString = + externalStringRegistryProvider.get().createProdString("Trending") + val KeywordTrendsTweetCountDescriptionString = + externalStringRegistryProvider.get().createProdString("KeywordTrends.tweetCountDescription") + + val NewsHeaderString = + externalStringRegistryProvider.get().createProdString("News.header") + val NewsFooterString = + externalStringRegistryProvider.get().createProdString("News.footer") } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.bazel index a56e3a1fd..562f36061 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.bazel @@ -4,11 +4,8 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "util/util-core/src/main/scala/com/twitter/conversions", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.scala index e43990507..63a3f9a10 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.scala @@ -2,6 +2,8 @@ package com.twitter.home_mixer.product.following.param import com.twitter.conversions.DurationOps._ import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.WhoToFollowModuleDisplayType +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowUserDisplayType +import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam @@ -11,6 +13,8 @@ import com.twitter.util.Duration object FollowingParam { val SupportedClientFSName = "following_supported_client" + val StaticParamValueZero = StaticParam(0) + val StaticParamValueFive = StaticParam(5) object ServerMaxResultsParam extends FSBoundedParam[Int]( @@ -26,32 +30,16 @@ object FollowingParam { default = true ) - object EnableAdsCandidatePipelineParam - extends FSParam[Boolean]( - name = "following_enable_ads", - default = true - ) - object EnableFlipInjectionModuleCandidatePipelineParam extends FSParam[Boolean]( name = "following_enable_flip_inline_injection_module", default = true ) - object FlipInlineInjectionModulePosition - extends FSBoundedParam[Int]( - name = "following_flip_inline_injection_module_position", - default = 0, - min = 0, - max = 1000 - ) - - object WhoToFollowPositionParam - extends FSBoundedParam[Int]( - name = "following_who_to_follow_position", - default = 5, - min = 0, - max = 99 + object EnablePostContextFeatureHydratorParam + extends FSParam[Boolean]( + name = "following_enable_post_context_feature_hydrator", + default = false ) object WhoToFollowMinInjectionIntervalParam @@ -71,6 +59,13 @@ object FollowingParam { enum = WhoToFollowModuleDisplayType ) + object WhoToFollowUserDisplayTypeIdParam + extends FSEnumParam[WhoToFollowUserDisplayType.type]( + name = "following_enable_who_to_follow_user_display_type_id", + default = WhoToFollowUserDisplayType.User, + enum = WhoToFollowUserDisplayType + ) + object WhoToFollowDisplayLocationParam extends FSParam[String]( name = "following_who_to_follow_display_location", @@ -82,4 +77,88 @@ object FollowingParam { name = "following_enable_fast_ads", default = true ) + + object EnableDependentAdsParam + extends FSParam[Boolean]( + name = "following_enable_dependent_ads", + default = true + ) + + object EnableNavigationInstructionParam + extends FSParam[Boolean]( + name = "following_enable_navigation_instruction", + default = false + ) + + object ClearCache { + object PtrEnableParam + extends FSParam[Boolean]( + name = "following_clear_cache_ptr_enable", + default = false + ) + + object ColdStartEnableParam + extends FSParam[Boolean]( + name = "following_clear_cache_cold_start_enable", + default = false + ) + + object WarmStartEnableParam + extends FSParam[Boolean]( + name = "following_clear_cache_warm_start_enable", + default = false + ) + + object ManualRefreshEnableParam + extends FSParam[Boolean]( + name = "following_clear_cache_manual_refresh_enable", + default = false + ) + + object NavigateEnableParam + extends FSParam[Boolean]( + name = "following_clear_cache_navigate_enable", + default = false + ) + + case object MinEntriesParam + extends FSBoundedParam[Int]( + name = "following_clear_cache_min_entries", + default = 10, + min = 0, + max = 35 + ) + } + + object Navigation { + object PtrEnableParam + extends FSParam[Boolean]( + name = "following_navigation_ptr_enable", + default = false + ) + + object ColdStartEnableParam + extends FSParam[Boolean]( + name = "following_navigation_cold_start_enable", + default = false + ) + + object WarmStartEnableParam + extends FSParam[Boolean]( + name = "following_navigation_warm_start_enable", + default = false + ) + + object ManualRefreshEnableParam + extends FSParam[Boolean]( + name = "following_navigation_manual_refresh_enable", + default = false + ) + + object NavigateEnableParam + extends FSParam[Boolean]( + name = "following_navigation_navigate_enable", + default = false + ) + } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.scala index df3b54801..c42a1df8e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.scala @@ -12,17 +12,27 @@ class FollowingParamConfig @Inject() () extends ProductParamConfig { override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableFollowingProduct override val supportedClientFSName: String = SupportedClientFSName - override val booleanFSOverrides = - Seq( - EnableFlipInjectionModuleCandidatePipelineParam, - EnableWhoToFollowCandidatePipelineParam, - EnableAdsCandidatePipelineParam, - EnableFastAds, - ) + override val booleanFSOverrides = Seq( + ClearCache.PtrEnableParam, + ClearCache.ColdStartEnableParam, + ClearCache.WarmStartEnableParam, + ClearCache.ManualRefreshEnableParam, + ClearCache.NavigateEnableParam, + EnableFlipInjectionModuleCandidatePipelineParam, + EnableWhoToFollowCandidatePipelineParam, + EnablePostContextFeatureHydratorParam, + EnableFastAds, + EnableDependentAdsParam, + EnableNavigationInstructionParam, + Navigation.PtrEnableParam, + Navigation.ColdStartEnableParam, + Navigation.WarmStartEnableParam, + Navigation.ManualRefreshEnableParam, + Navigation.NavigateEnableParam, + ) override val boundedIntFSOverrides = Seq( - FlipInlineInjectionModulePosition, - WhoToFollowPositionParam, + ClearCache.MinEntriesParam, ServerMaxResultsParam ) @@ -30,5 +40,8 @@ class FollowingParamConfig @Inject() () extends ProductParamConfig { override val boundedDurationFSOverrides = Seq(WhoToFollowMinInjectionIntervalParam) - override val enumFSOverrides = Seq(WhoToFollowDisplayTypeIdParam) + override val enumFSOverrides = Seq( + WhoToFollowDisplayTypeIdParam, + WhoToFollowUserDisplayTypeIdParam + ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel index 5f597c78b..76a056414 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel @@ -4,96 +4,81 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", - "finagle/finagle-memcached/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", + "communities-mixer/thrift/src/main/thrift:thrift-scala", + "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", + "geoduck/util/country", "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/entry_point_pivot", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/frame", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tweetypie", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/pivot", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/trends_events", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/communities_to_join", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline/selector", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/jetfuel_entry_point", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/job", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/recruiting_organization", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/cursor/timelines", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "src/java/com/twitter/search/common/util/lang", - "src/scala/com/twitter/suggests/controller_data", + "recruiting/candidate-service/src/main/thrift:thrift-scala", + "recruiting/src/main/thrift:thrift-scala", + "src/thrift/com/twitter/frigate/bookmarks:bookmarks-thrift-scala", "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", - "src/thrift/com/twitter/timelineservice:thrift-scala", - "stitch/stitch-tweetypie", + "strato/config/columns/events/entryPoint:entryPoint-strato-client", + "strato/config/columns/events/experiences/grok:grok-strato-client", + "strato/config/columns/trendsai/ordered:ordered-strato-client", + "strato/config/src/thrift/com/twitter/strato/columns/jetfuel:jetfuel-scala", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", ], exports = [ "src/thrift/com/twitter/timelines/render:thrift-scala", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala index 99ed0f584..e1be6b6a5 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala @@ -1,32 +1,33 @@ package com.twitter.home_mixer.product.for_you -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator import com.twitter.adserver.{thriftscala => ads} import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator.NoAdsTierFeature import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate +import com.twitter.home_mixer.functional_component.gate.ExcludeSyntheticUserGate import com.twitter.home_mixer.param.HomeGlobalParams import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.AdsNumOrganicItemsParam import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator +import com.twitter.product_mixer.component_library.gate.FeatureGate import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.StaticAdsDisplayLocationBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter -import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.gate.ParamNotGate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} import javax.inject.Inject import javax.inject.Singleton @@ -41,11 +42,11 @@ class ForYouAdsCandidatePipelineBuilder @Inject() ( private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouAds") - private val suggestType = st.SuggestType.Promoted + private val servedType = hmt.ServedType.ForYouPromoted private val clientEventInfoBuilder = ClientEventInfoBuilder( - component = InjectionScribeUtil.scribeComponent(suggestType).get, - detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) + component = servedType.originalName, + detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(servedType.name))) ) private val contextualTweetRefBuilder = ContextualTweetRefBuilder( @@ -65,9 +66,7 @@ class ForYouAdsCandidatePipelineBuilder @Inject() ( HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) - def build( - organicCandidatePipelines: Option[CandidateScope] = None - ): AdsCandidatePipelineConfig[ForYouQuery] = + def build(): AdsCandidatePipelineConfig[ForYouQuery] = adsCandidatePipelineConfigBuilder.build[ForYouQuery]( adsCandidateSource = adsCandidateSource, identifier = identifier, @@ -78,7 +77,9 @@ class ForYouAdsCandidatePipelineBuilder @Inject() ( name = "AdsDisableInjectionBasedOnUserRole", param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam ), - ExcludeSoftUserGate + ExcludeSoftUserGate, + ExcludeSyntheticUserGate, + FeatureGate.fromNegatedFeature(NoAdsTierFeature) ), filters = Seq(ValidAdImpressionIdFilter), postFilterFeatureHydration = Seq( @@ -89,6 +90,6 @@ class ForYouAdsCandidatePipelineBuilder @Inject() ( ), decorator = Some(decorator), alerts = alerts, - urtRequest = Some(true), + urtRequest = Some(true) ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala index ef65d7062..47f6b68a9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala @@ -2,7 +2,9 @@ package com.twitter.home_mixer.product.for_you import com.twitter.adserver.{thriftscala => ads} import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator.NoAdsTierFeature import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate +import com.twitter.home_mixer.functional_component.gate.ExcludeSyntheticUserGate import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.param.HomeGlobalParams @@ -10,6 +12,7 @@ import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafety import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder @@ -17,6 +20,7 @@ import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator +import com.twitter.product_mixer.component_library.gate.FeatureGate import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig @@ -30,8 +34,6 @@ import com.twitter.product_mixer.core.gate.ParamNotGate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} import javax.inject.Inject import javax.inject.Singleton @@ -47,13 +49,13 @@ class ForYouAdsDependentCandidatePipelineBuilder @Inject() ( private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouAdsDependent") - private val suggestType = st.SuggestType.Promoted + private val servedType = hmt.ServedType.ForYouPromoted private val MaxOrganicTweets = 35 private val clientEventInfoBuilder = ClientEventInfoBuilder( - component = InjectionScribeUtil.scribeComponent(suggestType).get, - detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) + component = servedType.originalName, + detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(servedType.name))) ) private val contextualTweetRefBuilder = ContextualTweetRefBuilder( @@ -95,6 +97,8 @@ class ForYouAdsDependentCandidatePipelineBuilder @Inject() ( param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam ), ExcludeSoftUserGate, + ExcludeSyntheticUserGate, + FeatureGate.fromNegatedFeature(NoAdsTierFeature), NonEmptyCandidatesGate(organicCandidatePipelines) ), filters = Seq(ValidAdImpressionIdFilter), diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouBookmarksCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouBookmarksCandidatePipelineConfig.scala new file mode 100644 index 000000000..7b8c373ec --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouBookmarksCandidatePipelineConfig.scala @@ -0,0 +1,88 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.frigate.bookmarks.{thriftscala => t} +import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselType +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator +import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter +import com.twitter.home_mixer.functional_component.filter.WeeklyBookmarkFilter +import com.twitter.home_mixer.functional_component.gate.BookmarksTimeGate +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.product.for_you.candidate_source.BookmarksCandidateSource +import com.twitter.home_mixer.product.for_you.param.ForYouParam.BookmarksModuleMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableBookmarksCandidatePipelineParam +import com.twitter.home_mixer.product.for_you.response_transformer.BookmarksResponseFeatureTransformer +import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelineservice.model.rich.EntityIdType +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouBookmarksCandidatePipelineConfig @Inject() ( + bookmarksTimeGate: BookmarksTimeGate, + bookmarksCandidateSource: BookmarksCandidateSource, + tweetypieFeatureHydrator: TweetypieFeatureHydrator, + tweetCarouselModuleCandidateDecorator: TweetCarouselModuleCandidateDecorator) + extends CandidatePipelineConfig[ + PipelineQuery, + Long, + t.BookmarkedTweet, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouBookmarks") + + override val supportedClientParam: Option[FSParam[Boolean]] = + Some(EnableBookmarksCandidatePipelineParam) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + DefinedUserIdGate, + RateLimitGate, + bookmarksTimeGate, + TimelinesPersistenceStoreLastInjectionGate( + BookmarksModuleMinInjectionIntervalParam, + EntityIdType.BookmarksModule + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[PipelineQuery, Long] = + query => query.getRequiredUserId + + override val candidateSource: BaseCandidateSource[Long, t.BookmarkedTweet] = + bookmarksCandidateSource + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[t.BookmarkedTweet] + ] = Seq(BookmarksResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + t.BookmarkedTweet, + TweetCandidate + ] = tweet => TweetCandidate(tweet.tweetId) + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] + ] = Seq(tweetypieFeatureHydrator) + + override val filters: Seq[Filter[PipelineQuery, TweetCandidate]] = + Seq(TweetHydrationFilter, WeeklyBookmarkFilter) + + override val decorator: Option[ + CandidateDecorator[PipelineQuery, TweetCandidate] + ] = Some(tweetCarouselModuleCandidateDecorator.build(TweetCarouselType.BookmarkedTweets)) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouCommunitiesToJoinCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouCommunitiesToJoinCandidatePipelineConfig.scala new file mode 100644 index 000000000..ebbe4bd02 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouCommunitiesToJoinCandidatePipelineConfig.scala @@ -0,0 +1,130 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.communities_mixer.{thriftscala => cmt} +import com.twitter.home_mixer.functional_component.decorator.urt.builder.FeedbackStrings +import com.twitter.home_mixer.functional_component.filter.DropMaxCandidatesFilter +import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.product.for_you.param.ForYouParam.CommunitiesToJoinDisplayTypeIdParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.CommunitiesToJoinMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableCommunitiesToJoinCandidatePipelineParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.MaxCommunitiesToJoinCandidatesParam +import com.twitter.product_mixer.component_library.candidate_source.communities.CommunitiesToJoinCandidateSource +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.IsAvailableToJoinFeature +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.communities.IsAvailableToJoinFeatureHydrator +import com.twitter.product_mixer.component_library.filter.FeatureFilter +import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate +import com.twitter.product_mixer.component_library.gate.communities.CommunitiesJoinLimitGate +import com.twitter.product_mixer.component_library.model.candidate.CommunityCandidate +import com.twitter.product_mixer.component_library.pipeline.candidate.communities_to_join.CommunitiesToJoinCandidateDecorator +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.HasPipelineCursor +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelineservice.model.rich.EntityIdType +import com.twitter.timelineservice.suggests.{thriftscala => st} +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +class ForYouCommunitiesToJoinCandidatePipelineConfig @Inject() ( + communitiesToJoinCandidateSource: CommunitiesToJoinCandidateSource, + communitiesJoinLimitGate: CommunitiesJoinLimitGate, + isAvailableToJoinFeatureHydrator: IsAvailableToJoinFeatureHydrator, + @ProductScoped stringCenterProvider: Provider[StringCenter], + externalStrings: HomeMixerExternalStrings, + feedbackStrings: FeedbackStrings) + extends CandidatePipelineConfig[ + PipelineQuery with HasPipelineCursor[_], + cmt.CommunitiesMixerRequest, + Long, + CommunityCandidate + ] { + + private val stringCenter = stringCenterProvider.get() + + private val MaxCommunities = 10 + + override val supportedClientParam: Option[FSParam[Boolean]] = Some( + EnableCommunitiesToJoinCandidatePipelineParam) + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouCommunitiesToJoin") + + private val IsAvailableToJoinFilterId = "IsAvailableToJoin" + + override val gates = Seq( + DefinedUserIdGate, + RateLimitGate, + TimelinesPersistenceStoreLastInjectionGate( + CommunitiesToJoinMinInjectionIntervalParam, + EntityIdType.CommunityModule + ), + DismissFatigueGate(st.SuggestType.CommunityToJoin, DismissInfoFeature), + communitiesJoinLimitGate + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + cmt.CommunitiesMixerRequest + ] = { query => + cmt.CommunitiesMixerRequest( + clientContext = ClientContextMarshaller(query.clientContext), + product = cmt.Product.CommunityRecs, + productContext = Some( + cmt.ProductContext.CommunityRecs( + cmt.CommunityRecs(displayFormat = cmt.DisplayFormat.Module))), + cursor = None, + maxResults = Some(MaxCommunities), + debugParams = None, + displayLocation = None + ) + } + + override val candidateSource: CandidateSource[cmt.CommunitiesMixerRequest, Long] = + communitiesToJoinCandidateSource + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[PipelineQuery, CommunityCandidate, _] + ] = Seq(isAvailableToJoinFeatureHydrator) + + override def filters: Seq[Filter[PipelineQuery, CommunityCandidate]] = Seq( + FeatureFilter + .fromFeature(FilterIdentifier(IsAvailableToJoinFilterId), IsAvailableToJoinFeature), + DropMaxCandidatesFilter(MaxCommunitiesToJoinCandidatesParam) + ) + + override val resultTransformer: CandidatePipelineResultsTransformer[Long, CommunityCandidate] = { + communityResult => CommunityCandidate(id = communityResult) + } + + override val decorator: Option[ + CandidateDecorator[PipelineQuery, CommunityCandidate] + ] = { + Some( + CommunitiesToJoinCandidateDecorator( + moduleDisplayTypeParam = CommunitiesToJoinDisplayTypeIdParam, + stringCenter = stringCenter, + headerString = externalStrings.CommunityToJoinHeaderString, + footerString = Some(externalStrings.CommunityToJoinFooterString), + seeLessOftenString = Some(feedbackStrings.seeLessOftenFeedbackString), + seeLessOftenConfirmationString = + Some(feedbackStrings.seeLessOftenConfirmationFeedbackString) + )) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouEntryPointPivotCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouEntryPointPivotCandidatePipelineBuilder.scala new file mode 100644 index 000000000..c68df7fc0 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouEntryPointPivotCandidatePipelineBuilder.scala @@ -0,0 +1,91 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.decorator.EntryPointPivotModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.StrEntryPointPivotCategoryText +import com.twitter.home_mixer.product.for_you.gate.FollowingSportsUserGate +import com.twitter.product_mixer.component_library.candidate_source.entry_point_pivot.EntryPointPivotCandidateSource +import com.twitter.product_mixer.component_library.candidate_source.entry_point_pivot.GrokEntryPointPivotCandidateSource +import com.twitter.product_mixer.component_library.model.candidate.pivot.PivotCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.Param +import com.twitter.timelines.render.{thriftscala => t} +import com.twitter.util.Duration +import javax.inject.Inject +import javax.inject.Singleton + +object ForYouEntryPointPivotCandidatePipelineBuilder { + sealed abstract class EntryPointPivotType() + + object EntryPointPivotType { + case object Events extends EntryPointPivotType() + + case object Grok extends EntryPointPivotType() + } +} +@Singleton +class ForYouEntryPointPivotCandidatePipelineBuilder @Inject() ( + entryPointPivotCandidateSource: EntryPointPivotCandidateSource, + grokEntryPointPivotCandidateSource: GrokEntryPointPivotCandidateSource) { + + import ForYouEntryPointPivotCandidatePipelineBuilder._ + + def build( + entryPointPivotType: EntryPointPivotType, + supportedClientParam: FSParam[Boolean], + pivotMinInjectionIntervalParam: Param[Duration] + ): ForYouEntryPointPivotCandidatePipelineConfig = + new ForYouEntryPointPivotCandidatePipelineConfig( + identifier = getCandidateIdentifier(entryPointPivotType), + supportedClientParam = Some(supportedClientParam), + candidateSource = getCandidateSource(entryPointPivotType), + decorator = Some(getDecorator(entryPointPivotType)), + pivotMinInjectionIntervalParam = pivotMinInjectionIntervalParam, + extraGates = getPipelineGates(entryPointPivotType) + ) + + private def getCandidateSource(entryPointPivotType: EntryPointPivotType): CandidateSource[ + StratoKeyView[String, Unit], + t.Pivot + ] = + entryPointPivotType match { + case EntryPointPivotType.Grok => + grokEntryPointPivotCandidateSource + case EntryPointPivotType.Events => + entryPointPivotCandidateSource + } + + private def getDecorator( + entryPointPivotType: EntryPointPivotType + ): CandidateDecorator[PipelineQuery, PivotCandidate] = + entryPointPivotType match { + case EntryPointPivotType.Grok => + EntryPointPivotModuleCandidateDecorator( + component = "grok_pivot", + headerText = StrEntryPointPivotCategoryText("") + ).moduleDecorator + case EntryPointPivotType.Events => + EntryPointPivotModuleCandidateDecorator( + component = "sport_entry_point", + headerText = StrEntryPointPivotCategoryText("Play now!") + ).moduleDecorator + } + + private def getPipelineGates(entryPointPivotType: EntryPointPivotType): Seq[Gate[PipelineQuery]] = + entryPointPivotType match { + case EntryPointPivotType.Grok => + Seq.empty + case EntryPointPivotType.Events => + Seq(FollowingSportsUserGate) + } + + private def getCandidateIdentifier( + entryPointPivotType: EntryPointPivotType + ): CandidatePipelineIdentifier = + CandidatePipelineIdentifier(s"ForYouEntryPointPivot${entryPointPivotType.toString}") +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouEntryPointPivotCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouEntryPointPivotCandidatePipelineConfig.scala new file mode 100644 index 000000000..2ea107e2a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouEntryPointPivotCandidatePipelineConfig.scala @@ -0,0 +1,81 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate +import com.twitter.product_mixer.component_library.model.candidate.pivot.PivotCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.HasPipelineCursor +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.Param +import com.twitter.timelines.render.{thriftscala => t} +import com.twitter.timelineservice.model.rich.EntityIdType +import com.twitter.util.Duration +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouEntryPointPivotCandidatePipelineConfig @Inject() ( + override val identifier: CandidatePipelineIdentifier, + override val supportedClientParam: Option[FSParam[Boolean]], + override val candidateSource: CandidateSource[ + StratoKeyView[String, Unit], + t.Pivot + ], + override val decorator: Option[CandidateDecorator[PipelineQuery, PivotCandidate]], + pivotMinInjectionIntervalParam: Param[Duration], + extraGates: Seq[Gate[PipelineQuery]]) + extends CandidatePipelineConfig[ + PipelineQuery with HasPipelineCursor[_], + StratoKeyView[String, Unit], + t.Pivot, + PivotCandidate + ] { + + override val gates = Seq( + DefinedUserIdGate, + RateLimitGate, + TimelinesPersistenceStoreLastInjectionGate( + pivotMinInjectionIntervalParam, + EntityIdType.EntryPointPivot + ) + ) ++ extraGates + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + StratoKeyView[String, Unit] + ] = { query => + StratoKeyView( + key = query.getCountryCode.getOrElse("US"), + view = None + ) + } + + override val resultTransformer: CandidatePipelineResultsTransformer[ + t.Pivot, + PivotCandidate + ] = { sourceResult => + PivotCandidate( + // We only expect one item per "messageprompt" entryNamespace per URT response. + // As a result id=0L will always be unique in the entryNamespace. + id = identifier.name, + url = sourceResult.url, + displayType = sourceResult.displayType, + titleText = sourceResult.titleText, + detailText = sourceResult.detailText, + image = sourceResult.image, + badge = sourceResult.badge, + categoryText = sourceResult.categoryText, + detailTextImage = sourceResult.detailTextImage, + element = None + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouExplorationTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouExplorationTweetsCandidatePipelineConfig.scala new file mode 100644 index 000000000..d0e402b1f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouExplorationTweetsCandidatePipelineConfig.scala @@ -0,0 +1,177 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator +import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter +import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature +import com.twitter.home_mixer.model.HomeFeatures.TLSOriginalTweetsWithConfirmedAuthorFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetAuthorFollowersFeature +import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetAuthorFeatureHydrator +import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetEngagementsFeatureHydrator +import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetAuthorFollowersFeatureHydrator +import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetEngagementCountsFeature +import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetEngagementCounts +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableExplorationTweetsCandidatePipelineParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.InNetworkExplorationTweetsMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.ExplorationTweetsMaxFollowerCountParam +import com.twitter.home_mixer.product.for_you.response_transformer.ExplorationTweetResponseFeatureTransformer +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.home_mixer.functional_component.gate.RecentlyServedByServedTypeGate +import com.twitter.home_mixer.product.for_you.gate.UserFollowingRangeGate +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Random + +@Singleton +class ForYouExplorationTweetsCandidatePipelineConfig @Inject() ( + tweetAuthorFeatureHydrator: TweetAuthorFeatureHydrator, + tweetEngagementsFeatureHydrator: TweetEngagementsFeatureHydrator, + tweetAuthorFollowersFeatureHydrator: TweetAuthorFollowersFeatureHydrator, + tweetypieFeatureHydrator: TweetypieFeatureHydrator, + homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) + extends CandidatePipelineConfig[ + ForYouQuery, + ForYouQuery, + Long, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouExplorationTweets") + + override val supportedClientParam: Option[FSParam[Boolean]] = Some( + EnableExplorationTweetsCandidatePipelineParam) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ForYouQuery, + ForYouQuery + ] = identity + + override val queryFeatureHydration: Seq[QueryFeatureHydrator[PipelineQuery]] = Seq( + tweetAuthorFeatureHydrator, + tweetEngagementsFeatureHydrator + ) + + override val queryFeatureHydrationPhase2: Seq[QueryFeatureHydrator[PipelineQuery]] = Seq( + tweetAuthorFollowersFeatureHydrator + ) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + RecentlyServedByServedTypeGate( + InNetworkExplorationTweetsMinInjectionIntervalParam, + hmt.ServedType.ForYouExploration + ), + UserFollowingRangeGate + ) + + override def candidateSource: CandidateSource[ForYouQuery, Long] = + PassthroughCandidateSource( + identifier = CandidateSourceIdentifier("ForYouExplorationTweets"), + candidateExtractor = { query => + val followedUserIds = query.features + .map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)) + .getOrElse(Seq.empty).toSet + + val servedAuthorIds = + query.features + .map(_.getOrElse(PersistenceEntriesFeature, Seq.empty)).getOrElse(Seq.empty) + .flatMap(_.entries.flatMap(_.sourceAuthorIds)) + .toSet + + val explorationAuthors = followedUserIds -- servedAuthorIds + + val engagementCounts = query.features + .map(_.getOrElse(TweetEngagementCountsFeature, Map.empty[Long, TweetEngagementCounts])) + .getOrElse(Map.empty) + + val maxFollowerCount = query.params(ExplorationTweetsMaxFollowerCountParam) + val tweetAuthorFollowers = query.features + .map(_.getOrElse(TweetAuthorFollowersFeature, Map.empty[Long, Option[Long]])) + .getOrElse(Map.empty) + + val groupedByAuthor = query.features + .map(_.getOrElse(TLSOriginalTweetsWithConfirmedAuthorFeature, Seq.empty)) + .toSeq.flatten.filter { + case (tweetId, authorId) => + explorationAuthors.contains(authorId) && { + val followerCount = tweetAuthorFollowers.get(tweetId).flatten.getOrElse(0L) + followerCount <= maxFollowerCount + } + } + .groupBy { case (_, authorId) => authorId } + + val bestPostsByAuthor = groupedByAuthor.flatMap { + case (authorId, candidates) => + val candidateWithEngagement = candidates.map { + case (tweetId, _) => + val engagement = engagementCounts.get(tweetId) + val totalEngagement = engagement + .map { counts => + counts.favoriteCount.getOrElse(0L) + + counts.replyCount.getOrElse(0L) + + counts.retweetCount.getOrElse(0L) + + counts.quoteCount.getOrElse(0L) + + counts.bookmarkCount.getOrElse(0L) + }.getOrElse(0L) + (tweetId, totalEngagement) + } + + if (candidateWithEngagement.nonEmpty) { + val bestPost = candidateWithEngagement.maxBy(_._2) + Some(bestPost._1) + } else None + } + + Random.shuffle(bestPostsByAuthor.toSeq).take(1) + } + ) + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[Long] + ] = Seq(ExplorationTweetResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[Long, TweetCandidate] = { + sourceResult => TweetCandidate(id = sourceResult) + } + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] + ] = Seq(tweetypieFeatureHydrator) + + override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = + Seq(TweetHydrationFilter) + + override val decorator: Option[CandidateDecorator[PipelineQuery, TweetCandidate]] = { + val clientEventInfoBuilder = + ClientEventInfoBuilder[PipelineQuery, TweetCandidate]( + hmt.ServedType.ForYouExploration.originalName) + + val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( + clientEventInfoBuilder = clientEventInfoBuilder, + feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder), + ) + + Some(UrtItemCandidateDecorator(tweetItemBuilder)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouJetfuelFrameFrameCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouJetfuelFrameFrameCandidatePipelineConfig.scala new file mode 100644 index 000000000..2f8f15c7c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouJetfuelFrameFrameCandidatePipelineConfig.scala @@ -0,0 +1,89 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.strato.columns.jetfuel.thriftscala.JetfuelRouteData +import com.twitter.home_mixer.product.for_you.candidate_source.JetfuelFrameCandidateSource +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableJetfuelFramePipelineParam +import com.twitter.home_mixer.product.for_you.gate.FollowingSportsUserGate +import com.twitter.product_mixer.component_library.pipeline.candidate.jetfuel_entry_point.JetfuelCandidateFeatureTransformer +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.frame.FrameCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.gate.FirstPageGate +import com.twitter.product_mixer.component_library.model.candidate.FrameCandidate +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.frame.JetfuelPayloadFeatureHydrator +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.HasPipelineCursor +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.strato.generated.client.events.entryPoint.JetfuelEntryPointByCountryClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouJetfuelFrameCandidatePipelineConfig @Inject() ( + jetfuelFrameCandidateSource: JetfuelFrameCandidateSource, + jetfuelPayloadFeatureHydrator: JetfuelPayloadFeatureHydrator) + extends CandidatePipelineConfig[ + PipelineQuery with HasPipelineCursor[_], + StratoKeyView[ + JetfuelEntryPointByCountryClientColumn.Key, + JetfuelEntryPointByCountryClientColumn.View + ], + JetfuelRouteData, + FrameCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouJetfuelFrame") + + private val FrameId = "ForYouJetfuelFrame" + + override val candidateSource: JetfuelFrameCandidateSource = jetfuelFrameCandidateSource + + override val postFilterFeatureHydration: Seq[ + BaseCandidateFeatureHydrator[PipelineQuery, FrameCandidate, _] + ] = Seq(jetfuelPayloadFeatureHydrator) + + override val gates: Seq[Gate[PipelineQuery with HasPipelineCursor[_]]] = Seq( + FirstPageGate, + ParamGate(name = "JetfuelFrameEnabled", param = EnableJetfuelFramePipelineParam), + FollowingSportsUserGate + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + StratoKeyView[ + JetfuelEntryPointByCountryClientColumn.Key, + JetfuelEntryPointByCountryClientColumn.View + ] + ] = { query => StratoKeyView(key = query.getCountryCode.getOrElse("US"), view = None) } + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[JetfuelRouteData] + ] = Seq(JetfuelCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + JetfuelRouteData, + FrameCandidate + ] = { sourceResult => FrameCandidate(id = sourceResult.route) } + + override val decorator: Option[ + CandidateDecorator[PipelineQuery, FrameCandidate] + ] = { + Some( + UrtItemCandidateDecorator( + FrameCandidateUrtItemBuilder( + frameId = FrameId, + clientEventInfoBuilder = None + ) + ) + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouKeywordTrendsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouKeywordTrendsCandidatePipelineConfig.scala new file mode 100644 index 000000000..00a90b0ce --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouKeywordTrendsCandidatePipelineConfig.scala @@ -0,0 +1,83 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.events.recos.thriftscala.GetUnfiedCandidatesRequest +import com.twitter.events.recos.{thriftscala => t} +import com.twitter.home_mixer.functional_component.decorator.KeywordTrendsModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.product.for_you.candidate_source.TrendCandidate +import com.twitter.home_mixer.product.for_you.candidate_source.UnifiedTrendsCandidateSource +import com.twitter.home_mixer.product.for_you.filter.PromotedTrendFilter +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableKeywordTrendsParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.KeywordTrendsModuleMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.MaxNumberKeywordTrendsParam +import com.twitter.home_mixer.product.for_you.query_transformer.UnifiedCandidatesQueryTransformer +import com.twitter.home_mixer.product.for_you.response_transformer.KeywordTrendsFeatureTransformer +import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate +import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelineservice.model.rich.EntityIdType +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouKeywordTrendsCandidatePipelineConfig @Inject() ( + unifiedTrendsCandidateSource: UnifiedTrendsCandidateSource, + keywordTrendsModuleCandidateDecorator: KeywordTrendsModuleCandidateDecorator) + extends CandidatePipelineConfig[ + PipelineQuery, + t.GetUnfiedCandidatesRequest, + TrendCandidate, + UnifiedTrendCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + "ForYouKeywordTrends") + + override val supportedClientParam: Option[FSParam[Boolean]] = Some(EnableKeywordTrendsParam) + + override val gates = Seq( + RateLimitGate, + DefinedUserIdGate, + TimelinesPersistenceStoreLastInjectionGate( + KeywordTrendsModuleMinInjectionIntervalParam, + EntityIdType.Trends + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + GetUnfiedCandidatesRequest + ] = UnifiedCandidatesQueryTransformer( + maxResultsParam = MaxNumberKeywordTrendsParam, + candidatePipelineIdentifier = identifier) + + override val candidateSource: CandidateSource[ + t.GetUnfiedCandidatesRequest, + TrendCandidate + ] = unifiedTrendsCandidateSource + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TrendCandidate] + ] = Seq(KeywordTrendsFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TrendCandidate, + UnifiedTrendCandidate + ] = { result => UnifiedTrendCandidate(id = result.candidate.trendName) } + + override val filters: Seq[Filter[PipelineQuery, UnifiedTrendCandidate]] = Seq(PromotedTrendFilter) + + override val decorator: Option[CandidateDecorator[PipelineQuery, UnifiedTrendCandidate]] = + Some(keywordTrendsModuleCandidateDecorator.moduleDecorator) + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouMixerPipelineConfig.scala new file mode 100644 index 000000000..c7ff07b10 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouMixerPipelineConfig.scala @@ -0,0 +1,584 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.clientapp.{thriftscala => ca} +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas +import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig +import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig +import com.twitter.home_mixer.candidate_pipeline.VerifiedPromptCandidatePipelineConfig +import com.twitter.home_mixer.functional_component.feature_hydrator._ +import com.twitter.home_mixer.functional_component.selector.UpdateConversationModuleId +import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails +import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration +import com.twitter.home_mixer.functional_component.side_effect._ +import com.twitter.home_mixer.param.HomeGlobalParams.EnableSSPAdsBrandSafetySettingsFeatureHydratorParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnableUserActionsShadowScribeParam +import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.product.for_you.feature_hydrator.DisplayedGrokTopicQueryFeatureHydrator +import com.twitter.home_mixer.product.for_you.feature_hydrator.FollowingSportsAccountsQueryFeatureHydrator +import com.twitter.home_mixer.product.for_you.feature_hydrator.TimelineServiceTweetsQueryFeatureHydrator +import com.twitter.home_mixer.product.for_you.feature_hydrator.ViewerHasJobRecommendationsFeatureHydrator +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableAdsDebugParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableEntryPointPivotParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFlipInjectionModuleCandidatePipelineParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFollowedGrokTopicsHydrationParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableForYouAppUpsellParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableForYouTimelineAdsSurface +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableForYouTopicSelectorParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableGrokEntryPointPivotParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EntryPointPivotMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.ExplorationTweetsTimelinePosition +import com.twitter.home_mixer.product.for_you.param.ForYouParam.ForYouAppUpsellJetfuelRouteParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.ForYouAppUpsellPosition +import com.twitter.home_mixer.product.for_you.param.ForYouParam.ForYouTopicSelectorJetfuelRouteParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.ForYouTopicSelectorPosition +import com.twitter.home_mixer.product.for_you.param.ForYouParam.GrokEntryPointPivotMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.GrokPivotModuleTimelinePosition +import com.twitter.home_mixer.product.for_you.param.ForYouParam.MaxNumberExplorationTweetsParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptTweetPositionParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.StaticParamValueFive +import com.twitter.home_mixer.product.for_you.param.ForYouParam.StaticParamValueZero +import com.twitter.home_mixer.product.for_you.param.ForYouParam.SuperbowlModuleTimelinePosition +import com.twitter.home_mixer.product.for_you.param.ForYouParam.TuneFeedTimelinePosition +import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselNumCandidates +import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselNumTweetCandidatesToDedupeAgainstParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselTimelinePosition +import com.twitter.home_mixer.product.for_you.selector.DebugUpdateSortAdsResult +import com.twitter.home_mixer.product.for_you.selector.RemoveDuplicateCandidatesOutsideModule +import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateFeatureKeysKafkaSideEffectBuilder +import com.twitter.home_mixer.product.for_you.side_effect.ServedStatsSideEffect +import com.twitter.home_mixer.product.for_you.side_effect.VideoServedStatsSideEffect +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.inject.annotations.Flag +import com.twitter.logpipeline.client.common.EventPublisher +import com.twitter.product_mixer.component_library.feature_hydrator.query.ads.SSPAdsBrandSafetySettingsFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptCandidatePipelineConfigBuilder +import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.selector.FlipPromptDynamicInsertionPosition +import com.twitter.product_mixer.component_library.pipeline.candidate.jetfuel_entry_point.JetfuelCandidatePipelineConfigBuilder +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidatePipelineConfig +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig +import com.twitter.product_mixer.component_library.selector.DropDuplicateCandidates +import com.twitter.product_mixer.component_library.selector.DropMaxCandidates +import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates +import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults +import com.twitter.product_mixer.component_library.selector.DropOrthogonalCandidates +import com.twitter.product_mixer.component_library.selector.IdAndClassDuplicationKey +import com.twitter.product_mixer.component_library.selector.InsertAppendResults +import com.twitter.product_mixer.component_library.selector.InsertDynamicPositionResults +import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults +import com.twitter.product_mixer.component_library.selector.PickFirstCandidateMerger +import com.twitter.product_mixer.component_library.selector.SelectConditionally +import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates +import com.twitter.product_mixer.component_library.selector.UpdateSortModuleItemCandidates +import com.twitter.product_mixer.component_library.selector.ads.AdsInjector +import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults +import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline +import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines +import com.twitter.product_mixer.core.functional_component.configapi.StaticParam +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller +import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller +import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline +import com.twitter.product_mixer.core.pipeline.FailOpenPolicy +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig +import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelines.render.{thriftscala => urt} +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +class ForYouMixerPipelineConfig @Inject() ( + forYouAdsCandidatePipelineBuilder: ForYouAdsCandidatePipelineBuilder, + forYouCommunitiesToJoinCandidatePipelineConfig: ForYouCommunitiesToJoinCandidatePipelineConfig, + forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, + forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, + forYouWhoToSubscribeCandidatePipelineConfigBuilder: ForYouWhoToSubscribeCandidatePipelineConfigBuilder, + forYouEntryPointPivotCandidatePipelineBuilder: ForYouEntryPointPivotCandidatePipelineBuilder, + forYouRecommendedJobsCandidatePipelineConfig: ForYouRecommendedJobsCandidatePipelineConfig, + forYouRecommendedRecruitingOrganizationsCandidatePipelineConfig: ForYouRecommendedRecruitingOrganizationsCandidatePipelineConfig, + forYouBookmarksCandidatePipelineConfig: ForYouBookmarksCandidatePipelineConfig, + forYouExplorationTweetsCandidatePipelineConfig: ForYouExplorationTweetsCandidatePipelineConfig, + forYouJetfuelFrameCandidatePipelineConfig: ForYouJetfuelFrameCandidatePipelineConfig, + forYouPinnedTweetsCandidatePipelineConfig: ForYouPinnedTweetsCandidatePipelineConfig, + forYouStoriesCandidatePipelineConfig: ForYouStoriesCandidatePipelineConfig, + flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, + forYouKeywordTrendsCandidatePipelineConfig: ForYouKeywordTrendsCandidatePipelineConfig, + forYouScoredVideoTweetsCandidatePipelineConfig: ForYouScoredVideoTweetsCandidatePipelineConfig, + forYouRelevancePromptCandidatePipelineConfig: ForYouRelevancePromptCandidatePipelineConfig, + editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, + forYouTuneFeedCandidatePipelineConfig: ForYouTuneFeedCandidatePipelineConfig, + newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], + forYouTweetPreviewsCandidatePipelineConfig: ForYouTweetPreviewsCandidatePipelineConfig, + verifiedPromptCandidatePipelineConfig: VerifiedPromptCandidatePipelineConfig, + jetfuelCandidatePipelineConfigBuilder: JetfuelCandidatePipelineConfigBuilder, + dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, + followingSportsAccountsQueryFeatureHydrator: FollowingSportsAccountsQueryFeatureHydrator, + gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, + impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator, + persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, + rateLimitQueryFeatureHydrator: RateLimitQueryFeatureHydrator, + requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery], + timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, + previewCreatorsQueryFeatureHydrator: PreviewCreatorsQueryFeatureHydrator, + sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, + userSubscriptionQueryFeatureHydrator: UserSubscriptionQueryFeatureHydrator, + displayedGrokTopicQueryFeatureHydrator: DisplayedGrokTopicQueryFeatureHydrator, + sspAdsBrandSafetySettingsFeatureHydrator: SSPAdsBrandSafetySettingsFeatureHydrator, + viewerHasJobRecommendationsFeatureHydrator: ViewerHasJobRecommendationsFeatureHydrator, + userActionsArrayByteQueryFeatureHydrator: UserActionsArrayByteQueryFeatureHydrator, + adsInjector: AdsInjector, + updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, + truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, + homeScribeServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, + servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, + clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], + externalStrings: HomeMixerExternalStrings, + @ProductScoped stringCenterProvider: Provider[StringCenter], + urtTransportMarshaller: UrtTransportMarshaller, + @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean, + statsReceiver: StatsReceiver) + extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { + + override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYou") + + private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep + + override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ForYouQuery]] = Seq( + rateLimitQueryFeatureHydrator, + requestQueryFeatureHydrator, + persistenceStoreQueryFeatureHydrator, + impressionBloomFilterQueryFeatureHydrator, + timelineServiceTweetsQueryFeatureHydrator, + previewCreatorsQueryFeatureHydrator, + sgsFollowedUsersQueryFeatureHydrator, + gizmoduckUserQueryFeatureHydrator, + viewerHasJobRecommendationsFeatureHydrator, + userSubscriptionQueryFeatureHydrator, + ParamGatedQueryFeatureHydrator( + EnableFollowedGrokTopicsHydrationParam, + displayedGrokTopicQueryFeatureHydrator + ), + ParamGatedQueryFeatureHydrator( + EnableSSPAdsBrandSafetySettingsFeatureHydratorParam, + sspAdsBrandSafetySettingsFeatureHydrator + ), + ParamGatedQueryFeatureHydrator( + EnableEntryPointPivotParam, + followingSportsAccountsQueryFeatureHydrator + ), + AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), + AsyncParamGatedQueryFeatureHydrator( + EnableUserActionsShadowScribeParam, + MixerPipelineConfig.resultSelectorsStep, + userActionsArrayByteQueryFeatureHydrator + ) + ) + + private val forYouAdsCandidatePipelineConfig = forYouAdsCandidatePipelineBuilder.build() + + private val forYouWhoToFollowCandidatePipelineConfig = + forYouWhoToFollowCandidatePipelineConfigBuilder.build() + + private val forYouWhoToSubscribeCandidatePipelineConfig = + forYouWhoToSubscribeCandidatePipelineConfigBuilder.build() + + private val flipPromptCandidatePipelineConfig = + flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery]( + supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) + ) + + private val forYouEventsEntryPointPivotCandidatePipelineConfig = + forYouEntryPointPivotCandidatePipelineBuilder.build( + entryPointPivotType = + ForYouEntryPointPivotCandidatePipelineBuilder.EntryPointPivotType.Events, + supportedClientParam = EnableEntryPointPivotParam, + pivotMinInjectionIntervalParam = EntryPointPivotMinInjectionIntervalParam + ) + + private val forYouGrokEntryPointPivotCandidatePipelineConfig = + forYouEntryPointPivotCandidatePipelineBuilder.build( + entryPointPivotType = ForYouEntryPointPivotCandidatePipelineBuilder.EntryPointPivotType.Grok, + supportedClientParam = EnableGrokEntryPointPivotParam, + pivotMinInjectionIntervalParam = GrokEntryPointPivotMinInjectionIntervalParam + ) + + private val forYouTopicSelectorCandidatePipelineConfig = { + jetfuelCandidatePipelineConfigBuilder.build[ForYouQuery]( + frameId = "ForYouTopicSelector", + identifier = CandidatePipelineIdentifier("ForYouTopicSelector"), + route = ForYouTopicSelectorJetfuelRouteParam, + gates = Seq( + ParamGate(name = "ForYouTopicSelector", param = EnableForYouTopicSelectorParam) + ) + ) + } + + private val forYouAppUpsellCandidatePipelineConfig = { + jetfuelCandidatePipelineConfigBuilder.build[ForYouQuery]( + frameId = "ForYouAppUpsell", + identifier = CandidatePipelineIdentifier("ForYouAppUpsell"), + route = ForYouAppUpsellJetfuelRouteParam, + gates = Seq( + ParamGate(name = "ForYouAppUpsell", param = EnableForYouAppUpsellParam) + ) + ) + } + + override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq( + forYouScoredTweetsCandidatePipelineConfig, + forYouAdsCandidatePipelineConfig, + forYouCommunitiesToJoinCandidatePipelineConfig, + forYouWhoToFollowCandidatePipelineConfig, + forYouWhoToSubscribeCandidatePipelineConfig, + forYouTweetPreviewsCandidatePipelineConfig, + forYouRecommendedJobsCandidatePipelineConfig, + forYouEventsEntryPointPivotCandidatePipelineConfig, + forYouGrokEntryPointPivotCandidatePipelineConfig, + forYouRecommendedRecruitingOrganizationsCandidatePipelineConfig, + forYouBookmarksCandidatePipelineConfig, + forYouExplorationTweetsCandidatePipelineConfig, + forYouPinnedTweetsCandidatePipelineConfig, + forYouStoriesCandidatePipelineConfig, + forYouScoredVideoTweetsCandidatePipelineConfig, + forYouTuneFeedCandidatePipelineConfig, + flipPromptCandidatePipelineConfig, + forYouKeywordTrendsCandidatePipelineConfig, + forYouJetfuelFrameCandidatePipelineConfig, + forYouRelevancePromptCandidatePipelineConfig, + forYouTopicSelectorCandidatePipelineConfig, + forYouAppUpsellCandidatePipelineConfig, + ) + + override val dependentCandidatePipelines: Seq[ + DependentCandidatePipelineConfig[ForYouQuery, _, _, _] + ] = Seq( + editedTweetsCandidatePipelineConfig, + newTweetsPillCandidatePipelineConfig, + verifiedPromptCandidatePipelineConfig + ) + + override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( + forYouScoredTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouCommunitiesToJoinCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouWhoToSubscribeCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouTweetPreviewsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouRecommendedJobsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouBookmarksCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouExplorationTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouPinnedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouStoriesCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouScoredVideoTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouTuneFeedCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouKeywordTrendsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouJetfuelFrameCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouEventsEntryPointPivotCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouGrokEntryPointPivotCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouRelevancePromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouTopicSelectorCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouAppUpsellCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + ) + + override val resultSelectors: Seq[Selector[ForYouQuery]] = Seq( + UpdateSortCandidates( + ordering = CandidatesUtil.scoreOrdering, + candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier + ), + UpdateSortModuleItemCandidates( + candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier, + ordering = CandidatesUtil.conversationModuleTweetsOrdering + ), + UpdateSortCandidates( + ordering = CandidatesUtil.scoreOrdering, + candidatePipeline = forYouPinnedTweetsCandidatePipelineConfig.identifier + ), + UpdateSortModuleItemCandidates( + ordering = CandidatesUtil.scoreOrdering, + candidatePipeline = forYouPinnedTweetsCandidatePipelineConfig.identifier + ), + UpdateConversationModuleId( + pipelineScope = SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier) + ), + RemoveDuplicateCandidatesOutsideModule( + pipelineScope = SpecificPipeline(forYouScoredVideoTweetsCandidatePipelineConfig.identifier), + candidatePipelinesOutsideModule = Set(forYouScoredTweetsCandidatePipelineConfig.identifier), + numCandidatesToCompareAgainst = VideoCarouselNumTweetCandidatesToDedupeAgainstParam + ), + RemoveDuplicateCandidatesOutsideModule( + pipelineScope = SpecificPipeline(forYouTuneFeedCandidatePipelineConfig.identifier), + candidatePipelinesOutsideModule = Set(forYouScoredTweetsCandidatePipelineConfig.identifier), + numCandidatesToCompareAgainst = StaticParam(10) + ), + DropMaxCandidates( + candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier, + maxSelectionsParam = ServerMaxResultsParam + ), + DropMaxCandidates( + candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, + maxSelectionsParam = MaxNumberReplaceInstructionsParam + ), + DropMaxCandidates( + candidatePipeline = forYouExplorationTweetsCandidatePipelineConfig.identifier, + maxSelectionsParam = MaxNumberExplorationTweetsParam + ), + DropMaxModuleItemCandidates( + candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, + maxModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MaxCandidatesSize) + ), + DropMaxModuleItemCandidates( + candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, + maxModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MaxCandidatesSize) + ), + DropMaxModuleItemCandidates( + candidatePipeline = forYouBookmarksCandidatePipelineConfig.identifier, + maxModuleItemsParam = StaticParam(10) + ), + DropMaxModuleItemCandidates( + candidatePipeline = forYouScoredVideoTweetsCandidatePipelineConfig.identifier, + maxModuleItemsParam = VideoCarouselNumCandidates + ), + DropMaxModuleItemCandidates( + candidatePipeline = forYouPinnedTweetsCandidatePipelineConfig.identifier, + maxModuleItemsParam = StaticParam(10) + ), + DropMaxModuleItemCandidates( + candidatePipeline = forYouTuneFeedCandidatePipelineConfig.identifier, + maxModuleItemsParam = StaticParam(3) + ), + DropMaxCandidates( + candidatePipeline = forYouPinnedTweetsCandidatePipelineConfig.identifier, + maxSelectionsParam = StaticParam(1) + ), + DropDuplicateCandidates( + pipelineScope = SpecificPipelines( + Set( + forYouScoredTweetsCandidatePipelineConfig.identifier, + forYouExplorationTweetsCandidatePipelineConfig.identifier + )), + duplicationKey = IdAndClassDuplicationKey, + mergeStrategy = PickFirstCandidateMerger + ), + InsertAppendResults( + candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier + ), + DropOrthogonalCandidates( + orthogonalCandidatePipelines = Seq( + forYouKeywordTrendsCandidatePipelineConfig.identifier, + forYouStoriesCandidatePipelineConfig.identifier, + forYouWhoToFollowCandidatePipelineConfig.identifier, + forYouWhoToSubscribeCandidatePipelineConfig.identifier, + forYouTweetPreviewsCandidatePipelineConfig.identifier, + forYouRecommendedJobsCandidatePipelineConfig.identifier, + forYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.identifier, + forYouCommunitiesToJoinCandidatePipelineConfig.identifier, + forYouBookmarksCandidatePipelineConfig.identifier, + ) + ), + DropOrthogonalCandidates( + orthogonalCandidatePipelines = Seq( + forYouEventsEntryPointPivotCandidatePipelineConfig.identifier, + forYouGrokEntryPointPivotCandidatePipelineConfig.identifier, + ) + ), + InsertFixedPositionResults( + candidatePipeline = forYouAppUpsellCandidatePipelineConfig.identifier, + positionParam = ForYouAppUpsellPosition + ), + InsertFixedPositionResults( + candidatePipeline = verifiedPromptCandidatePipelineConfig.identifier, + positionParam = StaticParamValueZero + ), + InsertDynamicPositionResults( + candidatePipeline = flipPromptCandidatePipelineConfig.identifier, + dynamicInsertionPosition = FlipPromptDynamicInsertionPosition(StaticParamValueZero) + ), + InsertFixedPositionResults( + candidatePipeline = forYouExplorationTweetsCandidatePipelineConfig.identifier, + positionParam = ExplorationTweetsTimelinePosition + ), + InsertFixedPositionResults( + candidatePipeline = forYouJetfuelFrameCandidatePipelineConfig.identifier, + positionParam = SuperbowlModuleTimelinePosition + ), + InsertFixedPositionResults( + candidatePipeline = forYouEventsEntryPointPivotCandidatePipelineConfig.identifier, + positionParam = SuperbowlModuleTimelinePosition + ), + InsertFixedPositionResults( + candidatePipeline = forYouGrokEntryPointPivotCandidatePipelineConfig.identifier, + positionParam = GrokPivotModuleTimelinePosition + ), + InsertFixedPositionResults( + candidatePipeline = forYouPinnedTweetsCandidatePipelineConfig.identifier, + positionParam = StaticParam(3) + ), + InsertFixedPositionResults( + candidatePipeline = forYouScoredVideoTweetsCandidatePipelineConfig.identifier, + positionParam = VideoCarouselTimelinePosition + ), + InsertFixedPositionResults( + candidatePipeline = forYouTuneFeedCandidatePipelineConfig.identifier, + positionParam = TuneFeedTimelinePosition + ), + InsertFixedPositionResults( + candidatePipeline = forYouTopicSelectorCandidatePipelineConfig.identifier, + positionParam = ForYouTopicSelectorPosition + ), + InsertFixedPositionResults( + candidatePipelines = Set( + forYouKeywordTrendsCandidatePipelineConfig.identifier, + forYouStoriesCandidatePipelineConfig.identifier, + forYouWhoToFollowCandidatePipelineConfig.identifier, + forYouWhoToSubscribeCandidatePipelineConfig.identifier, + forYouTweetPreviewsCandidatePipelineConfig.identifier, + forYouRecommendedJobsCandidatePipelineConfig.identifier, + forYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.identifier, + forYouCommunitiesToJoinCandidatePipelineConfig.identifier, + forYouBookmarksCandidatePipelineConfig.identifier, + ), + positionParam = StaticParamValueFive + ), + DropModuleTooFewModuleItemResults( + candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, + minModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MinCandidatesSize) + ), + DropModuleTooFewModuleItemResults( + candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, + minModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MinCandidatesSize) + ), + DropModuleTooFewModuleItemResults( + candidatePipeline = forYouBookmarksCandidatePipelineConfig.identifier, + minModuleItemsParam = StaticParam(2) + ), + DropModuleTooFewModuleItemResults( + candidatePipeline = forYouScoredVideoTweetsCandidatePipelineConfig.identifier, + minModuleItemsParam = StaticParam(3) + ), + DropModuleTooFewModuleItemResults( + candidatePipeline = forYouStoriesCandidatePipelineConfig.identifier, + minModuleItemsParam = StaticParam(3) + ), + DropModuleTooFewModuleItemResults( + candidatePipeline = forYouKeywordTrendsCandidatePipelineConfig.identifier, + minModuleItemsParam = StaticParam(3) + ), + SelectConditionally.paramNotGated( + InsertAdResults( + surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, + adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), + adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier + ), + EnableForYouTimelineAdsSurface + ), + SelectConditionally.paramGated( + InsertAdResults( + surfaceAreaName = AdsInjectionSurfaceAreas.ForYouTimeline, + adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.ForYouTimeline), + adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier + ), + EnableForYouTimelineAdsSurface + ), + SelectConditionally( + DebugUpdateSortAdsResult(forYouAdsCandidatePipelineConfig.identifier), + includeSelector = (query, _, _) => query.params(EnableAdsDebugParam) + ), + // This selector must come after the tweets are inserted into the results + UpdateNewTweetsPillDecoration( + pipelineScope = SpecificPipelines( + forYouScoredTweetsCandidatePipelineConfig.identifier, + newTweetsPillCandidatePipelineConfig.identifier + ), + stringCenter = stringCenterProvider.get(), + seeNewTweetsString = externalStrings.seeNewTweetsString, + tweetedString = externalStrings.tweetedString + ), + InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), + SelectConditionally( + selector = + InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), + includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) + ), + InsertFixedPositionResults( + candidatePipeline = forYouRelevancePromptCandidatePipelineConfig.identifier, + positionParam = RelevancePromptTweetPositionParam + ), + UpdateHomeClientEventDetails( + candidatePipelines = Set( + forYouScoredTweetsCandidatePipelineConfig.identifier, + forYouTweetPreviewsCandidatePipelineConfig.identifier, + forYouExplorationTweetsCandidatePipelineConfig.identifier, + forYouBookmarksCandidatePipelineConfig.identifier, + forYouPinnedTweetsCandidatePipelineConfig.identifier, + forYouScoredVideoTweetsCandidatePipelineConfig.identifier, + forYouTuneFeedCandidatePipelineConfig.identifier, + ) + ) + ) + + private val servedCandidateFeatureKeysKafkaSideEffect = + servedCandidateFeatureKeysKafkaSideEffectBuilder.build( + Set(forYouScoredTweetsCandidatePipelineConfig.identifier)) + + private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( + enableScribeClientEvents = enableScribeClientEvents, + logPipelinePublisher = clientEventsScribeEventPublisher, + injectedTweetsCandidatePipelineIdentifiers = + Seq(forYouScoredTweetsCandidatePipelineConfig.identifier), + adsCandidatePipelineIdentifier = Some(forYouAdsCandidatePipelineConfig.identifier), + whoToFollowCandidatePipelineIdentifier = + Some(forYouWhoToFollowCandidatePipelineConfig.identifier), + whoToSubscribeCandidatePipelineIdentifier = + Some(forYouWhoToSubscribeCandidatePipelineConfig.identifier), + forYouCommunitiesToJoinCandidatePipelineIdentifier = + Some(forYouCommunitiesToJoinCandidatePipelineConfig.identifier), + forYouRelevancePromptCandidatePipelineIdentifier = + Some(forYouRelevancePromptCandidatePipelineConfig.identifier) + ) + + override val resultSideEffects: Seq[PipelineResultSideEffect[ForYouQuery, Timeline]] = Seq( + updateTimelinesPersistenceStoreSideEffect, + truncateTimelinesPersistenceStoreSideEffect, + homeScribeClientEventSideEffect, + homeScribeServedCandidatesSideEffect, + servedCandidateFeatureKeysKafkaSideEffect, + ServedStatsSideEffect( + candidatePipelines = Set(forYouScoredTweetsCandidatePipelineConfig.identifier), + statsReceiver = statsReceiver + ), + VideoServedStatsSideEffect( + candidatePipelines = Set(forYouScoredTweetsCandidatePipelineConfig.identifier), + statsReceiver = statsReceiver + ), + ) + + override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = + ForYouResponseDomainMarshaller + + override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = + urtTransportMarshaller +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPinnedTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPinnedTweetsCandidatePipelineConfig.scala new file mode 100644 index 000000000..6c38ab4f6 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPinnedTweetsCandidatePipelineConfig.scala @@ -0,0 +1,114 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.decorator.PinnedTweetBroadcastCandidateDecorator +import com.twitter.home_mixer.functional_component.feature_hydrator.GizmoduckAuthorFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator +import com.twitter.home_mixer.functional_component.filter.CurrentPinnedTweetFilter +import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.product.for_you.candidate_source.BroadcastedPinnedTweetsCandidateSource +import com.twitter.home_mixer.product.for_you.candidate_source.PinnedTweetCandidate +import com.twitter.home_mixer.product.for_you.feature_hydrator.CurrentPinnedTweetFeatureHydrator +import com.twitter.home_mixer.product.for_you.filter.NotArticleFilter +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnablePinnedTweetsCandidatePipelineParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.PinnedTweetsModuleMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.response_transformer.PinnedTweetResponseFeatureTransformer +import com.twitter.home_mixer.product.for_you.scorer.PinnedTweetCandidateScorer +import com.twitter.product_mixer.component_library.filter.FeatureFilter +import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelineservice.model.rich.EntityIdType +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouPinnedTweetsCandidatePipelineConfig @Inject() ( + broadcastedPinnedTweetsCandidateSource: BroadcastedPinnedTweetsCandidateSource, + currentPinnedTweetFeatureHydrator: CurrentPinnedTweetFeatureHydrator, + gizmoduckAuthorFeatureHydrator: GizmoduckAuthorFeatureHydrator, + tweetypieFeatureHydrator: TweetypieFeatureHydrator, + pinnedTweetBroadcastCandidateDecorator: PinnedTweetBroadcastCandidateDecorator, +) extends CandidatePipelineConfig[ + ForYouQuery, + ForYouQuery, + PinnedTweetCandidate, + TweetCandidate + ] { + + private val InNetworkFilterId = "InNetwork" + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouPinnedTweets") + + override val supportedClientParam: Option[FSParam[Boolean]] = Some( + EnablePinnedTweetsCandidatePipelineParam) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + DefinedUserIdGate, + RateLimitGate, + TimelinesPersistenceStoreLastInjectionGate( + PinnedTweetsModuleMinInjectionIntervalParam, + EntityIdType.PinnedTweetsModule + ), + ) + + override val queryFeatureHydration: Seq[QueryFeatureHydrator[PipelineQuery]] = Seq() + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ForYouQuery, + ForYouQuery + ] = identity + + override def candidateSource: CandidateSource[ForYouQuery, PinnedTweetCandidate] = + broadcastedPinnedTweetsCandidateSource + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[PinnedTweetCandidate] + ] = Seq(PinnedTweetResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + PinnedTweetCandidate, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] + ] = Seq( + tweetypieFeatureHydrator, + InNetworkFeatureHydrator, + currentPinnedTweetFeatureHydrator, + gizmoduckAuthorFeatureHydrator + ) + + override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( + TweetHydrationFilter, + NotArticleFilter, + FeatureFilter.fromFeature(FilterIdentifier(InNetworkFilterId), InNetworkFeature), + CurrentPinnedTweetFilter + ) + + override def scorers: Seq[Scorer[ForYouQuery, TweetCandidate]] = Seq(PinnedTweetCandidateScorer) + + override val decorator: Option[ + CandidateDecorator[PipelineQuery, TweetCandidate] + ] = Some(pinnedTweetBroadcastCandidateDecorator) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala index e1cf2161c..c7ce7698e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala @@ -1,19 +1,17 @@ package com.twitter.home_mixer.product.for_you import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ForYouProductContext import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnablePushToHomeMixerPipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableScoredTweetsMixerPipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam import com.twitter.home_mixer.product.for_you.param.ForYouParamConfig import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer +import com.twitter.product_mixer.component_library.premarshaller.cursor.timelines.ChronologicalCursorUnmarshaller import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy import com.twitter.product_mixer.core.functional_component.common.alert.Alert import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert @@ -39,14 +37,13 @@ import com.twitter.timelines.render.{thriftscala => urt} import com.twitter.timelines.util.RequestCursorSerializer import com.twitter.util.Time import com.twitter.util.Try + import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouProductPipelineConfig @Inject() ( - forYouTimelineScorerMixerPipelineConfig: ForYouTimelineScorerMixerPipelineConfig, - forYouScoredTweetsMixerPipelineConfig: ForYouScoredTweetsMixerPipelineConfig, - forYouPushToHomeMixerPipelineConfig: ForYouPushToHomeMixerPipelineConfig, + forYouMixerPipelineConfig: ForYouMixerPipelineConfig, forYouParamConfig: ForYouParamConfig) extends ProductPipelineConfig[HomeMixerRequest, ForYouQuery, urt.TimelineResponse] { @@ -93,26 +90,14 @@ class ForYouProductPipelineConfig @Inject() ( debugOptions = debugOptions, deviceContext = context.deviceContext, seenTweetIds = context.seenTweetIds, - dspClientContext = context.dspClientContext, - pushToHomeTweetId = context.pushToHomeTweetId + dspClientContext = context.dspClientContext ) } - override val pipelines: Seq[PipelineConfig] = Seq( - forYouTimelineScorerMixerPipelineConfig, - forYouScoredTweetsMixerPipelineConfig, - forYouPushToHomeMixerPipelineConfig - ) + override val pipelines: Seq[PipelineConfig] = Seq(forYouMixerPipelineConfig) - override def pipelineSelector( - query: ForYouQuery - ): ComponentIdentifier = { - if (query.pushToHomeTweetId.isDefined && query.params(EnablePushToHomeMixerPipelineParam)) - forYouPushToHomeMixerPipelineConfig.identifier - else if (query.params(EnableScoredTweetsMixerPipelineParam)) - forYouScoredTweetsMixerPipelineConfig.identifier - else forYouTimelineScorerMixerPipelineConfig.identifier - } + override def pipelineSelector(query: ForYouQuery): ComponentIdentifier = + forYouMixerPipelineConfig.identifier override val alerts: Seq[Alert] = Seq( SuccessRateAlert( @@ -123,8 +108,8 @@ class ForYouProductPipelineConfig @Inject() ( LatencyAlert( notificationGroup = DefaultNotificationGroup, percentile = P99, - warnPredicate = TriggerIfLatencyAbove(2300.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(2800.millis, 15, 30) + warnPredicate = TriggerIfLatencyAbove(2800.millis, 15, 30), + criticalPredicate = TriggerIfLatencyAbove(3000.millis, 15, 30) ), ThroughputAlert( notificationGroup = DefaultNotificationGroup, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRecommendedJobsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRecommendedJobsCandidatePipelineConfig.scala new file mode 100644 index 000000000..deb801502 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRecommendedJobsCandidatePipelineConfig.scala @@ -0,0 +1,97 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.candidateservice.core_io.FetchJobRecommendationsView +import com.twitter.home_mixer.functional_component.decorator.urt.builder.FeedbackStrings +import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature +import com.twitter.home_mixer.model.HomeFeatures.ViewerHasJobRecommendationsEnabled +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.product.for_you.candidate_source.RecommendedJobsCandidateSource +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableRecommendedJobsParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.MaxRecommendedJobCandidatesParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.RecommendedJobMinInjectionIntervalParam +import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate +import com.twitter.product_mixer.component_library.gate.FeatureGate +import com.twitter.product_mixer.component_library.model.candidate.JobCandidate +import com.twitter.product_mixer.component_library.pipeline.candidate.job.RecommendedJobsCandidateDecorator +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelineservice.model.rich.EntityIdType +import com.twitter.timelineservice.suggests.{thriftscala => st} +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +class ForYouRecommendedJobsCandidatePipelineConfig @Inject() ( + recommendedJobsProductCandidateSource: RecommendedJobsCandidateSource, + @ProductScoped stringCenterProvider: Provider[StringCenter], + externalStrings: HomeMixerExternalStrings, + feedbackStrings: FeedbackStrings) + extends CandidatePipelineConfig[ + PipelineQuery, + StratoKeyView[Long, FetchJobRecommendationsView], + Long, + JobCandidate + ] { + + private val stringCenter = stringCenterProvider.get() + + override val supportedClientParam: Option[FSParam[Boolean]] = + Some(EnableRecommendedJobsParam) + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouRecommendedJobs") + + override val gates = Seq( + DefinedUserIdGate, + FeatureGate.fromFeature(ViewerHasJobRecommendationsEnabled), + RateLimitGate, + TimelinesPersistenceStoreLastInjectionGate( + RecommendedJobMinInjectionIntervalParam, + EntityIdType.JobModule + ), + DismissFatigueGate(st.SuggestType.Job, DismissInfoFeature) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + StratoKeyView[Long, FetchJobRecommendationsView] + ] = { query => + StratoKeyView( + query.getRequiredUserId, + FetchJobRecommendationsView(count = Some(query.params(MaxRecommendedJobCandidatesParam)))) + } + + override val candidateSource: BaseCandidateSource[ + StratoKeyView[Long, FetchJobRecommendationsView], + Long + ] = recommendedJobsProductCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + Long, + JobCandidate + ] = { jobResult => JobCandidate(id = jobResult) } + + override val decorator: Option[CandidateDecorator[PipelineQuery, JobCandidate]] = { + Some( + RecommendedJobsCandidateDecorator( + stringCenter = stringCenter, + headerString = externalStrings.RecommendedJobHeaderString, + footerString = externalStrings.RecommendedJobFooterString, + seeLessOftenString = feedbackStrings.seeLessOftenFeedbackString, + seeLessOftenConfirmationString = feedbackStrings.seeLessOftenConfirmationFeedbackString + )) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.scala new file mode 100644 index 000000000..71c55e7a3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.scala @@ -0,0 +1,102 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.decorator.urt.builder.FeedbackStrings +import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature +import com.twitter.home_mixer.model.HomeFeatures.ViewerHasRecruitingOrganizationRecommendationsEnabled +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.product.for_you.candidate_source.RecommendedRecruitingOrganizationsCandidateSource +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableRecommendedRecruitingOrganizationsParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.MaxRecommendedRecruitingOrganizationCandidatesParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.RecommendedRecruitingOrganizationMinInjectionIntervalParam +import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate +import com.twitter.product_mixer.component_library.gate.FeatureGate +import com.twitter.product_mixer.component_library.model.candidate.RecruitingOrganizationCandidate +import com.twitter.product_mixer.component_library.pipeline.candidate.recruiting_organization.RecommendedRecruitingOrganizationsCandidateDecorator +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.recruiting.organization.RecruitingOrganizationRecommendationsView +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelineservice.model.rich.EntityIdType +import com.twitter.timelineservice.suggests.{thriftscala => st} +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +class ForYouRecommendedRecruitingOrganizationsCandidatePipelineConfig @Inject() ( + recommendedRecruitingOrganizationsProductCandidateSource: RecommendedRecruitingOrganizationsCandidateSource, + @ProductScoped stringCenterProvider: Provider[StringCenter], + externalStrings: HomeMixerExternalStrings, + feedbackStrings: FeedbackStrings) + extends CandidatePipelineConfig[ + PipelineQuery, + StratoKeyView[Long, RecruitingOrganizationRecommendationsView], + Long, + RecruitingOrganizationCandidate + ] { + + private val stringCenter = stringCenterProvider.get() + + override val supportedClientParam: Option[FSParam[Boolean]] = + Some(EnableRecommendedRecruitingOrganizationsParam) + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouRecommendedRecruitingOrganizations") + + override val gates = Seq( + DefinedUserIdGate, + FeatureGate.fromFeature(ViewerHasRecruitingOrganizationRecommendationsEnabled), + RateLimitGate, + TimelinesPersistenceStoreLastInjectionGate( + RecommendedRecruitingOrganizationMinInjectionIntervalParam, + EntityIdType.RecruitingOrganizationModule + ), + DismissFatigueGate(st.SuggestType.RecruitingOrganization, DismissInfoFeature) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + StratoKeyView[Long, RecruitingOrganizationRecommendationsView] + ] = { query => + StratoKeyView( + query.getRequiredUserId, + RecruitingOrganizationRecommendationsView(count = + Some(query.params(MaxRecommendedRecruitingOrganizationCandidatesParam))) + ) + } + + override val candidateSource: BaseCandidateSource[ + StratoKeyView[Long, RecruitingOrganizationRecommendationsView], + Long + ] = + recommendedRecruitingOrganizationsProductCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + Long, + RecruitingOrganizationCandidate + ] = { orgResult => RecruitingOrganizationCandidate(id = orgResult) } + + override val decorator: Option[ + CandidateDecorator[PipelineQuery, RecruitingOrganizationCandidate] + ] = { + Some( + RecommendedRecruitingOrganizationsCandidateDecorator( + stringCenter = stringCenter, + headerString = externalStrings.RecommendedRecruitingOrganizationHeaderString, + footerString = externalStrings.RecommendedRecruitingOrganizationFooterString, + seeLessOftenString = feedbackStrings.seeLessOftenFeedbackString, + seeLessOftenConfirmationString = feedbackStrings.seeLessOftenConfirmationFeedbackString + )) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRelevancePromptCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRelevancePromptCandidatePipelineConfig.scala new file mode 100644 index 000000000..146605540 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRelevancePromptCandidatePipelineConfig.scala @@ -0,0 +1,66 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.decorator.urt.builder.RelevancePromptCandidateUrtItemBuilder +import com.twitter.home_mixer.functional_component.gate.PersistenceStoreDurationValidationGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptEnableParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptNegativeParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptNeutralParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptPositiveParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptTitleParam +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.model.candidate.RelevancePromptCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.candidate_source.StaticCandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelineservice.model.rich.EntityIdType +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouRelevancePromptCandidatePipelineConfig @Inject() () + extends CandidatePipelineConfig[ForYouQuery, Unit, Unit, RelevancePromptCandidate] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouRelevancePrompt") + + override val supportedClientParam: Option[FSParam[Boolean]] = Some(RelevancePromptEnableParam) + + override val gates = Seq( + PersistenceStoreDurationValidationGate(), + TimelinesPersistenceStoreLastInjectionGate( + RelevancePromptMinInjectionIntervalParam, + EntityIdType.Annotation + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ForYouQuery, Unit] = _ => Unit + + override def candidateSource: CandidateSource[Unit, Unit] = StaticCandidateSource[Unit]( + identifier = CandidateSourceIdentifier(identifier.name), + result = Seq(Unit) + ) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + Unit, + RelevancePromptCandidate + ] = _ => RelevancePromptCandidate(id = "feed-survey-prompt") + + override val decorator: Option[CandidateDecorator[ForYouQuery, RelevancePromptCandidate]] = Some( + UrtItemCandidateDecorator( + RelevancePromptCandidateUrtItemBuilder( + RelevancePromptTitleParam, + RelevancePromptPositiveParam, + RelevancePromptNegativeParam, + RelevancePromptNeutralParam + )) + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouResponseDomainMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouResponseDomainMarshaller.scala new file mode 100644 index 000000000..0fd2b0418 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouResponseDomainMarshaller.scala @@ -0,0 +1,84 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder +import com.twitter.home_mixer.model.ClearCacheIncludeInstruction +import com.twitter.home_mixer.model.NavigationIncludeInstruction +import com.twitter.home_mixer.model.request.DeviceContext.RequestContext +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam.ClearCache +import com.twitter.home_mixer.product.for_you.param.ForYouParam.Navigation +import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.NavigationInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedCursorIdSelector.TweetIdSelector +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder +import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller +import com.twitter.product_mixer.core.model.common.identifier.DomainMarshallerIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline +import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig + +object ForYouResponseDomainMarshaller extends DomainMarshaller[ForYouQuery, Timeline] { + + override val identifier: DomainMarshallerIdentifier = + DomainMarshallerIdentifier("ForYouResponse") + + override def apply( + query: ForYouQuery, + selections: Seq[CandidateWithDetails] + ): Timeline = { + val coldStart = + query.deviceContext.flatMap(_.requestContextValue).contains(RequestContext.Launch) + val retainViewportItems = + if (coldStart && query.params(ClearCache.ColdStartRetainViewportParam)) Some(true) else None + + val instructionBuilders = Seq( + ClearCacheInstructionBuilder( + includeInstruction = ClearCacheIncludeInstruction( + ClearCache.PtrEnableParam, + ClearCache.ColdStartEnableParam, + ClearCache.WarmStartEnableParam, + ClearCache.ManualRefreshEnableParam, + ClearCache.NavigateEnableParam, + ClearCache.MinEntriesParam + ), + retainViewportItems = retainViewportItems + ), + ReplaceEntryInstructionBuilder(ReplaceAllEntries), + // excludes alert, cover, and replace candidates + AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), + ShowAlertInstructionBuilder(), + ShowCoverInstructionBuilder(), + NavigationInstructionBuilder( + NavigationIncludeInstruction( + Navigation.PtrEnableParam, + Navigation.ColdStartEnableParam, + Navigation.WarmStartEnableParam, + Navigation.ManualRefreshEnableParam, + Navigation.NavigateEnableParam + )) + ) + + val topCursorBuilder = OrderedTopCursorBuilder(TweetIdSelector) + val bottomCursorBuilder = OrderedBottomCursorBuilder(TweetIdSelector) + + val scribeConfigBuilder = + StaticTimelineScribeConfigBuilder(TimelineScribeConfig(page = Some("for_you"), None, None)) + val metadataBuilder = UrtMetadataBuilder(scribeConfigBuilder = Some(scribeConfigBuilder)) + + val domainMarshaller = UrtDomainMarshaller( + instructionBuilders = instructionBuilders, + metadataBuilder = Some(metadataBuilder), + cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) + ) + + domainMarshaller(query, selections) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala index d7716e190..cd3867925 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala @@ -1,36 +1,20 @@ package com.twitter.home_mixer.product.for_you -import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetSocialContextBuilder +import com.twitter.home_mixer.functional_component.decorator.ForYouTweetCandidateDecorator +import com.twitter.home_mixer.functional_component.feature_hydrator.BasketballContextFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter +import com.twitter.home_mixer.functional_component.feature_hydrator.PostContextFeatureHydrator import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter -import com.twitter.home_mixer.functional_component.gate.SupportedLanguagesGate -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.param.HomeGlobalParams.EnableBasketballContextFeatureHydratorParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnablePostContextFeatureHydratorParam import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetWithConversationMetadata import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetsProductCandidateSource import com.twitter.home_mixer.product.for_you.feature_hydrator.FocalTweetFeatureHydrator -import com.twitter.home_mixer.product.for_you.filter.SocialContextFilter import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableScoredTweetsCandidatePipelineParam import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ManualModuleId -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder -import com.twitter.product_mixer.component_library.filter.FeatureFilter -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.communities.CommunityNamesFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator @@ -41,11 +25,9 @@ import com.twitter.product_mixer.core.functional_component.transformer.Candidate import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.decider.DeciderParam +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator + import javax.inject.Inject import javax.inject.Singleton @@ -54,10 +36,11 @@ class ForYouScoredTweetsCandidatePipelineConfig @Inject() ( scoredTweetsProductCandidateSource: ScoredTweetsProductCandidateSource, focalTweetFeatureHydrator: FocalTweetFeatureHydrator, namesFeatureHydrator: NamesFeatureHydrator, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, + communityNamesFeatureHydrator: CommunityNamesFeatureHydrator, + basketballContextFeatureHydrator: BasketballContextFeatureHydrator, + postContextFeatureHydrator: PostContextFeatureHydrator, invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, - homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder) + forYouTweetCandidateDecorator: ForYouTweetCandidateDecorator) extends CandidatePipelineConfig[ ForYouQuery, ForYouQuery, @@ -68,19 +51,11 @@ class ForYouScoredTweetsCandidatePipelineConfig @Inject() ( override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouScoredTweets") - private val TweetypieHydratedFilterId = "TweetypieHydrated" - private val QuotedTweetDroppedFilterId = "QuotedTweetDropped" - private val OutOfNetworkNSFWFilterId = "OutOfNetworkNSFW" - private val ConversationModuleNamespace = EntryNamespace("home-conversation") - - override val gates: Seq[Gate[ForYouQuery]] = Seq(SupportedLanguagesGate) + override val gates: Seq[Gate[ForYouQuery]] = Seq(RateLimitGate) override val candidateSource: CandidateSource[ForYouQuery, ScoredTweetWithConversationMetadata] = scoredTweetsProductCandidateSource - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(EnableScoredTweetsCandidatePipelineParam) - override val queryTransformer: CandidatePipelineQueryTransformer[ForYouQuery, ForYouQuery] = identity @@ -93,60 +68,29 @@ class ForYouScoredTweetsCandidatePipelineConfig @Inject() ( TweetCandidate ] = { sourceResults => TweetCandidate(sourceResults.tweetId) } - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq(InNetworkFeatureHydrator, namesFeatureHydrator, tweetypieFeatureHydrator) - override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( - FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(QuotedTweetDroppedFilterId), - shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } - ), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(OutOfNetworkNSFWFilterId), - shouldKeepCandidate = { features => - features.getOrElse(InNetworkFeature, false) || - !features.getOrElse(IsNsfwFeature, false) - } - ), - SocialContextFilter, - invalidSubscriptionTweetFilter, - InvalidConversationModuleFilter + invalidSubscriptionTweetFilter ) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq(focalTweetFeatureHydrator) - - override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = { - val clientEventInfoBuilder = HomeClientEventInfoBuilder() - - val tweetItemBuilder = TweetCandidateUrtItemBuilder( - clientEventInfoBuilder = clientEventInfoBuilder, - socialContextBuilder = Some(homeTweetSocialContextBuilder), - timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) - ) - - val tweetDecorator = UrtItemCandidateDecorator(tweetItemBuilder) - - val moduleBuilder = TimelineModuleBuilder( - entryNamespace = ConversationModuleNamespace, - clientEventInfoBuilder = clientEventInfoBuilder, - moduleIdGeneration = ManualModuleId(0L), - displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), - metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) - ) + ] = Seq( + focalTweetFeatureHydrator, + InNetworkFeatureHydrator, + namesFeatureHydrator, + communityNamesFeatureHydrator, + ParamGatedBulkCandidateFeatureHydrator( + EnableBasketballContextFeatureHydratorParam, + basketballContextFeatureHydrator + ), + ParamGatedBulkCandidateFeatureHydrator( + EnablePostContextFeatureHydratorParam, + postContextFeatureHydrator + ), + ) - Some( - UrtMultipleModulesDecorator( - urtItemCandidateDecorator = tweetDecorator, - moduleBuilder = moduleBuilder, - groupByKey = (_, _, candidateFeatures) => - candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) - )) - } + override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = + Some(forYouTweetCandidateDecorator.build()) override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala index eafc09352..0f3d95366 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala @@ -1,7 +1,38 @@ package com.twitter.home_mixer.product.for_you +import com.twitter.home_mixer.model.GrokTopics.GrokCategoryIdToNameMap import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.model.PhoenixPredictedBookmarkScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedDwellScoreFeature +import com.twitter.home_mixer.model.candidate_source.SourceSignal +import com.twitter.home_mixer.model.PhoenixPredictedFavoriteScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedGoodProfileClickScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedNegativeFeedbackV2ScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedOpenLinkScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedReplyScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedRetweetScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedScreenshotScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedShareScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedVideoQualityViewScoreFeature +import com.twitter.home_mixer.model.PredictedDwellScoreFeature +import com.twitter.home_mixer.model.PredictedFavoriteScoreFeature +import com.twitter.home_mixer.model.PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature +import com.twitter.home_mixer.model.PredictedGoodClickConvoDescUamGt2ScoreFeature +import com.twitter.home_mixer.model.PredictedGoodProfileClickScoreFeature +import com.twitter.home_mixer.model.PredictedNegativeFeedbackV2ScoreFeature +import com.twitter.home_mixer.model.PredictedReplyEngagedByAuthorScoreFeature +import com.twitter.home_mixer.model.PredictedReplyScoreFeature +import com.twitter.home_mixer.model.PredictedRetweetScoreFeature +import com.twitter.home_mixer.model.PredictedShareScoreFeature +import com.twitter.home_mixer.model.PredictedVideoQualityViewScoreFeature import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetWithConversationMetadata +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityNameFeature +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder @@ -11,7 +42,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ba import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType import com.twitter.timelines.render.{thriftscala => tl} -import com.twitter.timelineservice.suggests.{thriftscala => tls} object ForYouScoredTweetsResponseFeatureTransformer extends CandidateFeatureTransformer[ScoredTweetWithConversationMetadata] { @@ -27,9 +57,12 @@ object ForYouScoredTweetsResponseFeatureTransformer AuthorIsGoldVerifiedFeature, AuthorIsGrayVerifiedFeature, AuthorIsLegacyVerifiedFeature, + AuthorFollowersFeature, ConversationModuleFocalTweetIdFeature, ConversationModuleIdFeature, DirectedAtUserIdFeature, + DebugStringFeature, + SourceSignalFeature, ExclusiveConversationAuthorIdFeature, FullScoringSucceededFeature, FavoritedByUserIdsFeature, @@ -38,20 +71,70 @@ object ForYouScoredTweetsResponseFeatureTransformer InReplyToTweetIdFeature, InReplyToUserIdFeature, IsAncestorCandidateFeature, + IsNsfw, IsReadFromCacheFeature, IsRetweetFeature, - PerspectiveFilteredLikedByUserIdsFeature, + CommunityIdFeature, + CommunityNameFeature, + ListIdFeature, + ListNameFeature, + LocationIdFeature, + PredictionRequestIdFeature, QuotedTweetIdFeature, QuotedUserIdFeature, SGSValidFollowedByUserIdsFeature, SGSValidLikedByUserIdsFeature, + ValidLikedByUserIdsFeature, ScoreFeature, + ServedTypeFeature, SourceTweetIdFeature, SourceUserIdFeature, - StreamToKafkaFeature, - SuggestTypeFeature, TopicContextFunctionalityTypeFeature, - TopicIdSocialContextFeature + TopicIdSocialContextFeature, + TweetLanguageFeature, + TweetTextFeature, + TweetTypeMetricsFeature, + UserActionsSizeFeature, + UserActionsContainsExplicitSignalsFeature, + VisibilityReason, + ViralContentCreatorFeature, + GrokContentCreatorFeature, + GorkContentCreatorFeature, + HasVideoFeature, + VideoDurationMsFeature, + TweetMediaIdsFeature, + GrokAnnotationsFeature, + GrokTopCategoryFeature, + GrokIsGoreFeature, + GrokIsNsfwFeature, + GrokIsSpamFeature, + GrokIsViolentFeature, + GrokIsLowQualityFeature, + GrokIsOcrFeature, + PredictedDwellScoreFeature, + PredictedFavoriteScoreFeature, + PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, + PredictedGoodClickConvoDescUamGt2ScoreFeature, + PredictedGoodProfileClickScoreFeature, + PredictedNegativeFeedbackV2ScoreFeature, + PredictedReplyEngagedByAuthorScoreFeature, + PredictedReplyScoreFeature, + PredictedRetweetScoreFeature, + PredictedShareScoreFeature, + PredictedVideoQualityViewScoreFeature, + PhoenixPredictedDwellScoreFeature, + PhoenixPredictedFavoriteScoreFeature, + PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, + PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature, + PhoenixPredictedGoodProfileClickScoreFeature, + PhoenixPredictedNegativeFeedbackV2ScoreFeature, + PhoenixPredictedReplyScoreFeature, + PhoenixPredictedRetweetScoreFeature, + PhoenixPredictedShareScoreFeature, + PhoenixPredictedVideoQualityViewScoreFeature, + PhoenixPredictedOpenLinkScoreFeature, + PhoenixPredictedScreenshotScoreFeature, + PhoenixPredictedBookmarkScoreFeature ) override def transform(input: ScoredTweetWithConversationMetadata): FeatureMap = @@ -63,12 +146,23 @@ object ForYouScoredTweetsResponseFeatureTransformer .add(AuthorIsGrayVerifiedFeature, input.authorIsGrayVerified.getOrElse(false)) .add(AuthorIsLegacyVerifiedFeature, input.authorIsLegacyVerified.getOrElse(false)) .add(AuthorIsCreatorFeature, input.authorIsCreator.getOrElse(false)) + .add(AuthorFollowersFeature, input.authorFollowers) + .add(CommunityIdFeature, input.communityId) + .add(CommunityNameFeature, input.communityName) .add(ConversationModuleIdFeature, input.conversationId) .add(ConversationModuleFocalTweetIdFeature, input.conversationFocalTweetId) .add(DirectedAtUserIdFeature, input.directedAtUserId) + .add(DebugStringFeature, input.debugString) + .add( + SourceSignalFeature, + input.sourceSignal.map { ss => + SourceSignal(ss.id, ss.signalType, ss.signalEntity, ss.authorId) + } + ) .add(ExclusiveConversationAuthorIdFeature, input.exclusiveConversationAuthorId) .add(SGSValidLikedByUserIdsFeature, input.sgsValidLikedByUserIds.getOrElse(Seq.empty)) .add(SGSValidFollowedByUserIdsFeature, input.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) + .add(ValidLikedByUserIdsFeature, input.validLikedByUserIds.getOrElse(Seq.empty)) .add(FavoritedByUserIdsFeature, input.sgsValidLikedByUserIds.getOrElse(Seq.empty)) .add(FollowedByUserIdsFeature, input.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) .add(FullScoringSucceededFeature, true) @@ -78,16 +172,17 @@ object ForYouScoredTweetsResponseFeatureTransformer .add(IsAncestorCandidateFeature, input.conversationFocalTweetId.exists(_ != input.tweetId)) .add(IsReadFromCacheFeature, input.isReadFromCache.getOrElse(false)) .add(IsRetweetFeature, input.sourceTweetId.isDefined) - .add( - PerspectiveFilteredLikedByUserIdsFeature, - input.perspectiveFilteredLikedByUserIds.getOrElse(Seq.empty)) + .add(IsNsfw, input.isNsfw) + .add(ListIdFeature, input.listId) + .add(ListNameFeature, input.listName) + .add(LocationIdFeature, input.locationId) + .add(PredictionRequestIdFeature, input.predictionRequestId) .add(QuotedTweetIdFeature, input.quotedTweetId) .add(QuotedUserIdFeature, input.quotedUserId) .add(ScoreFeature, input.score) .add(SourceTweetIdFeature, input.sourceTweetId) .add(SourceUserIdFeature, input.sourceUserId) - .add(StreamToKafkaFeature, input.streamToKafka.getOrElse(false)) - .add(SuggestTypeFeature, input.suggestType.orElse(Some(tls.SuggestType.Undefined))) + .add(ServedTypeFeature, input.servedType) .add( TopicContextFunctionalityTypeFeature, input.topicFunctionalityType.collect { @@ -99,5 +194,96 @@ object ForYouScoredTweetsResponseFeatureTransformer } ) .add(TopicIdSocialContextFeature, input.topicId) + .add(TweetLanguageFeature, input.tweetLanguage) + .add(TweetTextFeature, input.tweetText) + .add(TweetTypeMetricsFeature, input.tweetTypeMetrics) + .add(UserActionsSizeFeature, input.userActionsSize) + .add( + UserActionsContainsExplicitSignalsFeature, + input.userActionsContainsExplicitSignals.getOrElse(false)) + .add(VisibilityReason, input.visibilityReason) + .add(ViralContentCreatorFeature, input.viralContentCreatorFeature.getOrElse(false)) + .add(GrokContentCreatorFeature, input.grokContentCreatorFeature.getOrElse(false)) + .add(GorkContentCreatorFeature, input.gorkContentCreatorFeature.getOrElse(false)) + .add(HasVideoFeature, input.hasVideoFeature.getOrElse(false)) + .add(VideoDurationMsFeature, input.videoDurationMsFeature) + .add(TweetMediaIdsFeature, input.mediaIds.getOrElse(Seq.empty)) + .add(GrokAnnotationsFeature, input.grokAnnotations) + .add(GrokIsGoreFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isGore))) + .add(GrokIsNsfwFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isNsfw))) + .add(GrokIsSpamFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isSpam))) + .add(GrokIsViolentFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isViolent))) + .add(GrokIsLowQualityFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isLowQuality))) + .add(GrokIsOcrFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isOcr))) + .add( + GrokTopCategoryFeature, + input.grokAnnotations.flatMap { annotations => + annotations.categoryScores.flatMap { scores => + val validCategories = scores.collect { + case (category, score) + if category.forall(_.isDigit) && + GrokCategoryIdToNameMap.contains(category.toLong) => + (category.toLong, score) + } + if (validCategories.nonEmpty) Some(validCategories.maxBy(_._2)._1) else None + } + } + ) + .add(PredictedDwellScoreFeature, input.predictedScores.flatMap(_.dwellScore)) + .add(PredictedFavoriteScoreFeature, input.predictedScores.flatMap(_.favoriteScore)) + .add( + PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, + input.predictedScores.flatMap(_.goodClickConvoDescFavoritedOrRepliedScore)) + .add( + PredictedGoodClickConvoDescUamGt2ScoreFeature, + input.predictedScores.flatMap(_.goodClickConvoDescUamGt2Score)) + .add( + PredictedGoodProfileClickScoreFeature, + input.predictedScores.flatMap(_.goodProfileClickScore)) + .add( + PredictedNegativeFeedbackV2ScoreFeature, + input.predictedScores.flatMap(_.negativeFeedbackV2Score)) + .add( + PredictedReplyEngagedByAuthorScoreFeature, + input.predictedScores.flatMap(_.replyEngagedByAuthorScore)) + .add(PredictedReplyScoreFeature, input.predictedScores.flatMap(_.replyScore)) + .add(PredictedRetweetScoreFeature, input.predictedScores.flatMap(_.retweetScore)) + .add(PredictedShareScoreFeature, input.predictedScores.flatMap(_.shareScore)) + .add( + PredictedVideoQualityViewScoreFeature, + input.predictedScores.flatMap(_.videoQualityViewScore)) + .add(PhoenixPredictedDwellScoreFeature, input.phoenixPredictedScores.flatMap(_.dwellScore)) + .add( + PhoenixPredictedFavoriteScoreFeature, + input.phoenixPredictedScores.flatMap(_.favoriteScore)) + .add( + PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, + input.phoenixPredictedScores.flatMap(_.goodClickConvoDescFavoritedOrRepliedScore)) + .add( + PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature, + input.phoenixPredictedScores.flatMap(_.goodClickConvoDescUamGt2Score)) + .add( + PhoenixPredictedGoodProfileClickScoreFeature, + input.phoenixPredictedScores.flatMap(_.goodProfileClickScore)) + .add( + PhoenixPredictedNegativeFeedbackV2ScoreFeature, + input.phoenixPredictedScores.flatMap(_.negativeFeedbackV2Score)) + .add(PhoenixPredictedReplyScoreFeature, input.phoenixPredictedScores.flatMap(_.replyScore)) + .add( + PhoenixPredictedRetweetScoreFeature, + input.phoenixPredictedScores.flatMap(_.retweetScore)) + .add(PhoenixPredictedShareScoreFeature, input.phoenixPredictedScores.flatMap(_.shareScore)) + .add( + PhoenixPredictedVideoQualityViewScoreFeature, + input.phoenixPredictedScores.flatMap(_.videoQualityViewScore)) + .add( + PhoenixPredictedOpenLinkScoreFeature, + input.phoenixPredictedScores.flatMap(_.openLinkScore)) + .add( + PhoenixPredictedScreenshotScoreFeature, + input.phoenixPredictedScores.flatMap(_.screenshotScore)) + .add( + PhoenixPredictedBookmarkScoreFeature, + input.phoenixPredictedScores.flatMap(_.bookmarkScore)) .build() } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredVideoTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredVideoTweetsCandidatePipelineConfig.scala new file mode 100644 index 000000000..518b649bd --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredVideoTweetsCandidatePipelineConfig.scala @@ -0,0 +1,102 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.decorator.VideoCarouselModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator +import com.twitter.home_mixer.functional_component.filter.ConsistentAspectRatioFilter +import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.product.for_you.candidate_source.ScoredVideoTweetCandidate +import com.twitter.home_mixer.product.for_you.candidate_source.ScoredVideoTweetsCategorizedProductCandidateSource +import com.twitter.home_mixer.product.for_you.candidate_source.ScoredVideoTweetsProductCandidateSource +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableScoredVideoTweetsCandidatePipelineParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselAllowHorizontalVideos +import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselAllowVerticalVideos +import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselEnableFooterParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoTweetsModuleMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.response_transformer.ScoredVideoTweetResponseFeatureTransformer +import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelineservice.model.rich.EntityIdType +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouScoredVideoTweetsCandidatePipelineConfig @Inject() ( + scoredVideoTweetsProductCandidateSource: ScoredVideoTweetsProductCandidateSource, + scoredVideoTweetsCategorizedProductCandidateSource: ScoredVideoTweetsCategorizedProductCandidateSource, + tweetypieFeatureHydrator: TweetypieFeatureHydrator, + videoCarouselModuleCandidateDecorator: VideoCarouselModuleCandidateDecorator) + extends CandidatePipelineConfig[ + ForYouQuery, + ForYouQuery, + ScoredVideoTweetCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouScoredVideoTweets") + + override val supportedClientParam: Option[FSParam[Boolean]] = Some( + EnableScoredVideoTweetsCandidatePipelineParam) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + DefinedUserIdGate, + RateLimitGate, + TimelinesPersistenceStoreLastInjectionGate( + VideoTweetsModuleMinInjectionIntervalParam, + EntityIdType.VideoCarouselModule + ), + ) + + override val queryFeatureHydration: Seq[QueryFeatureHydrator[PipelineQuery]] = Seq() + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ForYouQuery, + ForYouQuery + ] = identity + + override def candidateSource: CandidateSource[ForYouQuery, ScoredVideoTweetCandidate] = + scoredVideoTweetsCategorizedProductCandidateSource +// scoredVideoTweetsProductCandidateSource + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[ScoredVideoTweetCandidate] + ] = Seq(ScoredVideoTweetResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + ScoredVideoTweetCandidate, + TweetCandidate + ] = { sourceResult => + TweetCandidate(id = sourceResult.tweetId) + } + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] + ] = Seq(tweetypieFeatureHydrator) + + override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( + TweetHydrationFilter, + ConsistentAspectRatioFilter( + allowVerticalVideosParam = VideoCarouselAllowVerticalVideos, + allowHorizontalVideosParam = VideoCarouselAllowHorizontalVideos) + ) + + override val decorator: Option[ + CandidateDecorator[PipelineQuery, TweetCandidate] + ] = Some(videoCarouselModuleCandidateDecorator.build(VideoCarouselEnableFooterParam)) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouStoriesCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouStoriesCandidatePipelineConfig.scala new file mode 100644 index 000000000..86acbfb61 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouStoriesCandidatePipelineConfig.scala @@ -0,0 +1,69 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.decorator.StoriesModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.product.for_you.candidate_source.StoriesModuleCandidateSource +import com.twitter.home_mixer.product.for_you.candidate_source.StoryCandidate +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTrendsParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.TrendsModuleMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.response_transformer.StoriesModuleResponseFeatureTransformer +import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate +import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelineservice.model.rich.EntityIdType +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouStoriesCandidatePipelineConfig @Inject() ( + storiesModule: StoriesModuleCandidateSource, + storiesModuleCandidateDecorator: StoriesModuleCandidateDecorator) + extends CandidatePipelineConfig[ + PipelineQuery, + Long, + StoryCandidate, + UnifiedTrendCandidate + ] { + + override val supportedClientParam: Option[FSParam[Boolean]] = Some(EnableTrendsParam) + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouStories") + + override val gates = Seq( + RateLimitGate, + DefinedUserIdGate, + TimelinesPersistenceStoreLastInjectionGate( + TrendsModuleMinInjectionIntervalParam, + EntityIdType.Trends + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + Long + ] = { query => query.getRequiredUserId } + + override def candidateSource: BaseCandidateSource[Long, StoryCandidate] = storiesModule + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[StoryCandidate] + ] = Seq(StoriesModuleResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + StoryCandidate, + UnifiedTrendCandidate + ] = { story => UnifiedTrendCandidate(id = story.id.toString) } + + override val decorator: Option[CandidateDecorator[PipelineQuery, UnifiedTrendCandidate]] = + Some(storiesModuleCandidateDecorator.moduleDecorator) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTuneFeedCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTuneFeedCandidatePipelineConfig.scala new file mode 100644 index 000000000..46bb8e3f0 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTuneFeedCandidatePipelineConfig.scala @@ -0,0 +1,101 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.geoduck.util.country.CountryInfo +import com.twitter.home_mixer.functional_component.decorator.TuneFeedModuleCandidateDecorator +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator +import com.twitter.home_mixer.functional_component.filter.PreviouslySeenTweetsFilter +import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.model.HomeFeatures.CurrentDisplayedGrokTopicFeature +import com.twitter.home_mixer.product.for_you.candidate_source.TuneFeedCandidateSource +import com.twitter.home_mixer.product.for_you.candidate_source.TuneFeedRequest +import com.twitter.home_mixer.product.for_you.gate.TuneFeedModuleGate +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTuneFeedCandidatePipelineParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.TuneFeedModuleMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.response_transformer.TuneFeedFeatureTransformer +import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelineservice.model.rich.EntityIdType +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouTuneFeedCandidatePipelineConfig @Inject() ( + tuneFeedCandidateSource: TuneFeedCandidateSource, + tweetypieFeatureHydrator: TweetypieFeatureHydrator, + tuneFeedModuleDecorator: TuneFeedModuleCandidateDecorator) + extends CandidatePipelineConfig[ + ForYouQuery, + TuneFeedRequest, + TweetCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouTuneFeed") + + override val supportedClientParam: Option[FSParam[Boolean]] = Some( + EnableTuneFeedCandidatePipelineParam) + + override val gates: Seq[Gate[ForYouQuery]] = Seq( + DefinedUserIdGate, + RateLimitGate, + TuneFeedModuleGate, + TimelinesPersistenceStoreLastInjectionGate( + TuneFeedModuleMinInjectionIntervalParam, + EntityIdType.TuneFeedModule + ), + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ForYouQuery, + TuneFeedRequest + ] = { query => + // Safe .get, enforced in TuneFeedModuleGate + val displayedGrokTopic = + query.features.get.getOrElse(CurrentDisplayedGrokTopicFeature, None).get + + TuneFeedRequest( + grokTopic = displayedGrokTopic._1, + language = query.getLanguageCode, + placeId = query.getCountryCode.flatMap(CountryInfo.lookupByCode).map(_.placeIdLong) + ) + } + + override def candidateSource: BaseCandidateSource[TuneFeedRequest, TweetCandidate] = + tuneFeedCandidateSource + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetCandidate] + ] = Seq(TuneFeedFeatureTransformer) + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] + ] = Seq(tweetypieFeatureHydrator) + + override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( + TweetHydrationFilter, + PreviouslySeenTweetsFilter + ) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetCandidate, + TweetCandidate + ] = identity + + override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = Some( + tuneFeedModuleDecorator) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala index c60278bfb..191ca63df 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala @@ -1,27 +1,32 @@ package com.twitter.home_mixer.product.for_you -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.filter.DropMaxCandidatesFilter import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetPreviewsFilter +import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter +import com.twitter.home_mixer.functional_component.gate.PersistenceStoreDurationValidationGate +import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.AuthorEnabledPreviewsFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature +import com.twitter.home_mixer.product.for_you.feature_hydrator.ArticlePreviewTextFeatureHydrator import com.twitter.home_mixer.product.for_you.feature_hydrator.AuthorEnabledPreviewsFeatureHydrator import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetPreviewTweetypieCandidateFeatureHydrator import com.twitter.home_mixer.product.for_you.filter.TweetPreviewTextFilter import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableArticlePreviewTextHydrationParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTweetPreviewsCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsMaxCandidatesParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.query_transformer.TweetPreviewsQueryTransformer import com.twitter.home_mixer.product.for_you.response_transformer.TweetPreviewResponseFeatureTransformer import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.candidate_source.earlybird.EarlybirdTweetCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsFeature import com.twitter.product_mixer.component_library.filter.FeatureFilter import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate @@ -43,19 +48,17 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.injection.scribe.InjectionScribeUtil import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.timelineservice.suggests.{thriftscala => st} - import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouTweetPreviewsCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, + earlybirdTweetCandidateSource: EarlybirdTweetCandidateSource, authorEnabledPreviewsFeatureHydrator: AuthorEnabledPreviewsFeatureHydrator, tweetPreviewsQueryTransformer: TweetPreviewsQueryTransformer, tweetPreviewTweetypieCandidateFeatureHydrator: TweetPreviewTweetypieCandidateFeatureHydrator, + articlePreviewTextFeatureHydrator: ArticlePreviewTextFeatureHydrator, homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) extends CandidatePipelineConfig[ ForYouQuery, @@ -71,9 +74,10 @@ class ForYouTweetPreviewsCandidatePipelineConfig @Inject() ( override val gates: Seq[Gate[ForYouQuery]] = { Seq( + RateLimitGate, + PersistenceStoreDurationValidationGate(), TimelinesPersistenceStoreLastInjectionGate( TweetPreviewsMinInjectionIntervalParam, - PersistenceEntriesFeature, EntityIdType.TweetPreview ), NonEmptySeqFeatureGate(PreviewCreatorsFeature) @@ -88,7 +92,7 @@ class ForYouTweetPreviewsCandidatePipelineConfig @Inject() ( override val candidateSource: CandidateSourceWithExtractedFeatures[ eb.EarlybirdRequest, eb.ThriftSearchResult - ] = earlybirdCandidateSource + ] = earlybirdTweetCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[eb.ThriftSearchResult] @@ -106,21 +110,31 @@ class ForYouTweetPreviewsCandidatePipelineConfig @Inject() ( tweetPreviewTweetypieCandidateFeatureHydrator, ) + override val preFilterFeatureHydrationPhase2: Seq[ + BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] + ] = Seq( + ParamGatedCandidateFeatureHydrator( + candidateFeatureHydrator = articlePreviewTextFeatureHydrator, + enabledParam = EnableArticlePreviewTextHydrationParam)) + override val filters: Seq[ Filter[ForYouQuery, TweetCandidate] ] = Seq( PreviouslyServedTweetPreviewsFilter, - FeatureFilter - .fromFeature(FilterIdentifier("TweetPreviewVisibilityFiltering"), IsHydratedFeature), + TweetHydrationFilter, FeatureFilter .fromFeature(FilterIdentifier("AuthorEnabledPreviews"), AuthorEnabledPreviewsFeature), TweetPreviewTextFilter, + // NOTE: since earlybird returns candidates sorted by score, this will filter + // for the top scoring candidates. Should be updated in the future to not assume order of + // candidates from earlybird. DropMaxCandidatesFilter(TweetPreviewsMaxCandidatesParam) ) override val decorator: Option[CandidateDecorator[PipelineQuery, TweetCandidate]] = { - val component = InjectionScribeUtil.scribeComponent(st.SuggestType.TweetPreview).get - val clientEventInfoBuilder = ClientEventInfoBuilder[PipelineQuery, TweetCandidate](component) + val clientEventInfoBuilder = + ClientEventInfoBuilder[PipelineQuery, TweetCandidate]( + hmt.ServedType.ForYouTweetPreview.originalName) val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( clientEventInfoBuilder = clientEventInfoBuilder, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala index 856f1fa6b..434180431 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala @@ -2,17 +2,20 @@ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToFollowFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate +import com.twitter.home_mixer.functional_component.gate.RateLimitGate +import com.twitter.home_mixer.functional_component.gate.TestUserProbabilisticGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableWhoToFollowCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowDisplayLocationParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowDisplayTypeIdParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowUserDisplayTypeIdParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.ParamWhoToFollowUserDisplayTypeBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfigBuilder import com.twitter.product_mixer.core.functional_component.gate.Gate @@ -24,16 +27,18 @@ import javax.inject.Singleton @Singleton class ForYouWhoToFollowCandidatePipelineConfigBuilder @Inject() ( whoToFollowArmCandidatePipelineConfigBuilder: WhoToFollowArmCandidatePipelineConfigBuilder, - homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder) { + homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder, + testUserProbabilisticGate: TestUserProbabilisticGate) { def build(): WhoToFollowArmCandidatePipelineConfig[ForYouQuery] = { val gates: Seq[Gate[ForYouQuery]] = Seq( + RateLimitGate, TimelinesPersistenceStoreLastInjectionGate( WhoToFollowMinInjectionIntervalParam, - PersistenceEntriesFeature, EntityIdType.WhoToFollow ), - DismissFatigueGate(SuggestType.WhoToFollow, DismissInfoFeature) + DismissFatigueGate(SuggestType.WhoToFollow, DismissInfoFeature), + testUserProbabilisticGate ) whoToFollowArmCandidatePipelineConfigBuilder.build[ForYouQuery]( @@ -45,6 +50,8 @@ class ForYouWhoToFollowCandidatePipelineConfigBuilder @Inject() ( ParamWhoToFollowModuleDisplayTypeBuilder(WhoToFollowDisplayTypeIdParam), feedbackActionInfoBuilder = Some(homeWhoToFollowFeedbackActionInfoBuilder), excludedUserIdsFeature = Some(WhoToFollowExcludedUserIdsFeature), + userDisplayTypeBuilder = + ParamWhoToFollowUserDisplayTypeBuilder(WhoToFollowUserDisplayTypeIdParam), displayLocationParam = WhoToFollowDisplayLocationParam ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala index bd4437b7c..b6aab66ee 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala @@ -2,9 +2,9 @@ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToSubscribeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate +import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableWhoToSubscribeCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribeDisplayTypeIdParam @@ -26,9 +26,9 @@ class ForYouWhoToSubscribeCandidatePipelineConfigBuilder @Inject() ( def build(): WhoToSubscribeCandidatePipelineConfig[ForYouQuery] = { val gates: Seq[Gate[ForYouQuery]] = Seq( + RateLimitGate, TimelinesPersistenceStoreLastInjectionGate( WhoToSubscribeMinInjectionIntervalParam, - PersistenceEntriesFeature, EntityIdType.WhoToSubscribe ), DismissFatigueGate(SuggestType.WhoToSubscribe, DismissInfoFeature) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel index 540823800..ebc486d92 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel @@ -4,19 +4,28 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", + "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/search:earlybird-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "recruiting/candidate-service/src/main/thrift:thrift-scala", + "recruiting/src/main/thrift:thrift-scala", + "src/thrift/com/twitter/frigate/bookmarks:bookmarks-thrift-scala", + "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "stitch/stitch-timelineservice/src/main/scala", + "strato/config/columns/events/entryPoint:entryPoint-strato-client", + "strato/config/columns/frigate/bookmarks:bookmarks-strato-client", + "strato/config/columns/recruiting/api/user:user-strato-client", + "strato/config/columns/recruiting/candidate_service:candidate_service-strato-client", + "strato/config/columns/trends/trip:trip-strato-client", + "strato/config/columns/trendsai/ordered:ordered-strato-client", + "strato/config/src/thrift/com/twitter/strato/columns/jetfuel:jetfuel-scala", + "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", ], exports = [ "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BookmarksCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BookmarksCandidateSource.scala new file mode 100644 index 000000000..45c2f4468 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BookmarksCandidateSource.scala @@ -0,0 +1,46 @@ +package com.twitter.home_mixer.product.for_you.candidate_source + +import com.twitter.frigate.bookmarks.thriftscala.BookmarkedTweet +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.frigate.bookmarks.WeeklyBookmarksOnUserClientColumn + +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Random + +@Singleton +class BookmarksCandidateSource @Inject() ( + weeklyBookmarksOnUserClientColumn: WeeklyBookmarksOnUserClientColumn) + extends CandidateSource[ + Long, + BookmarkedTweet + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("Bookmarks") + + private val MaxCandidates = 20 + private val fetcher = weeklyBookmarksOnUserClientColumn.fetcher + + // Remove duplicates and unbookmarked tweets + private def filterValidBookmarks(bookmarks: Seq[BookmarkedTweet]): Seq[BookmarkedTweet] = { + bookmarks + .groupBy(_.tweetId) + .collect { + case (_, duplicatedBookmarks) + if !duplicatedBookmarks.exists(_.unbookmark.getOrElse(false)) => + duplicatedBookmarks.head + }.toSeq + } + + override def apply(userId: Long): Stitch[Seq[BookmarkedTweet]] = { + fetcher.fetch(userId, ()).map { res => + res.v + .map { bookmarks => + val filteredBookmarks = filterValidBookmarks(bookmarks.bookmarkedTweets) + Random.shuffle(filteredBookmarks).take(MaxCandidates) + }.getOrElse(Seq.empty) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BroadcastedPinnedTweetsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BroadcastedPinnedTweetsCandidateSource.scala new file mode 100644 index 000000000..14cdfc3cf --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BroadcastedPinnedTweetsCandidateSource.scala @@ -0,0 +1,66 @@ +package com.twitter.home_mixer.product.for_you.candidate_source + +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import com.twitter.stitch.timelineservice.TimelineService +import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilterItem +import com.twitter.timelineservice.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +case class PinnedTweetCandidate(tweetId: Long, userId: Option[Long]) + +@Singleton +class BroadcastedPinnedTweetsCandidateSource @Inject() ( + timelineService: TimelineService) + extends CandidateSource[ + ForYouQuery, + PinnedTweetCandidate + ] { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("BroadcastedPinnedTweets") + + private val MaxTimelineServiceTweets = 100 + private val MaxTweetsForHydration = 20 + + override def apply(query: ForYouQuery): Stitch[Seq[PinnedTweetCandidate]] = { + val userId = query.getRequiredUserId + val timelineQueryOptions = t.TimelineQueryOptions( + contextualUserId = Some(userId) + ) + + val timelineServiceQuery = t.TimelineQuery( + timelineType = t.TimelineType.PinnedTweets, + timelineId = userId, + maxCount = MaxTimelineServiceTweets.toShort, + cursor2 = None, + options = Some(timelineQueryOptions), + timelineId2 = Some(t.TimelineId(t.TimelineType.PinnedTweets, userId)), + ) + + timelineService + .getTimeline(timelineServiceQuery).map { timeline => + timeline.entries.collect { + case t.TimelineEntry.Tweet(tweet) => + PinnedTweetCandidate(tweet.statusId, tweet.userId) + } + }.map { candidates => + val bloomFilterSeq = query.features.map(_.get(ImpressionBloomFilterFeature)).get + val bloomFilters = + bloomFilterSeq.entries.map(ImpressionBloomFilterItem.fromThrift(_).bloomFilter) + + val seenTweetIds = query.seenTweetIds.getOrElse(Seq.empty).toSet + + candidates + .filter { pinnedTweetCandidate => + !seenTweetIds.contains(pinnedTweetCandidate.tweetId) && + !bloomFilters.exists(filter => filter.mayContain(pinnedTweetCandidate.tweetId)) + } + .take(MaxTweetsForHydration) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/JetfuelFrameCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/JetfuelFrameCandidateSource.scala new file mode 100644 index 000000000..8ce66d8d6 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/JetfuelFrameCandidateSource.scala @@ -0,0 +1,37 @@ +package com.twitter.home_mixer.product.for_you.candidate_source + +import com.twitter.strato.columns.jetfuel.thriftscala.JetfuelRouteData +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.events.entryPoint.JetfuelEntryPointByCountryClientColumn + +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Strato column to fetch entry point for sports to display in search results. + */ +@Singleton +class JetfuelFrameCandidateSource @Inject() ( + entryPointClientColumn: JetfuelEntryPointByCountryClientColumn) + extends StratoKeyViewFetcherSource[ + JetfuelEntryPointByCountryClientColumn.Key, + JetfuelEntryPointByCountryClientColumn.View, + JetfuelEntryPointByCountryClientColumn.Value, + JetfuelRouteData + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("JetfuelFrame") + + override val fetcher: Fetcher[ + JetfuelEntryPointByCountryClientColumn.Key, + JetfuelEntryPointByCountryClientColumn.View, + JetfuelEntryPointByCountryClientColumn.Value, + ] = entryPointClientColumn.fetcher + + override def stratoResultTransformer( + stratoKey: JetfuelEntryPointByCountryClientColumn.Key, + stratoResult: JetfuelEntryPointByCountryClientColumn.Value + ): Seq[JetfuelRouteData] = stratoResult +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/RecommendedJobsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/RecommendedJobsCandidateSource.scala new file mode 100644 index 000000000..e9c259076 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/RecommendedJobsCandidateSource.scala @@ -0,0 +1,37 @@ +package com.twitter.home_mixer.product.for_you.candidate_source + +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherSource +import com.twitter.strato.generated.client.recruiting.candidate_service.JobRecommendationCandidatesOnUserClientColumn +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.candidateservice.core_io.FetchJobRecommendationsView +import com.twitter.strato.client.Client + +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class RecommendedJobsCandidateSource @Inject() ( + @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client) + extends StratoKeyViewFetcherSource[ + Long, + FetchJobRecommendationsView, + JobRecommendationCandidatesOnUserClientColumn.Value, + Long + ] { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("RecommendedJobs") + + val fetcher = stratoClient.fetcher[ + Long, + FetchJobRecommendationsView, + JobRecommendationCandidatesOnUserClientColumn.Value + ](JobRecommendationCandidatesOnUserClientColumn.Path) + + override protected def stratoResultTransformer( + stratoKey: Long, + stratoResult: JobRecommendationCandidatesOnUserClientColumn.Value + ): Seq[Long] = stratoResult.map(_.apiJob) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/RecommendedRecruitingOrganizationsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/RecommendedRecruitingOrganizationsCandidateSource.scala new file mode 100644 index 000000000..7c1bf89c7 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/RecommendedRecruitingOrganizationsCandidateSource.scala @@ -0,0 +1,37 @@ +package com.twitter.home_mixer.product.for_you.candidate_source + +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherSource +import com.twitter.strato.generated.client.recruiting.api.user.RecruitingOrganizationRecommendationsOnUserClientColumn +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.recruiting.organization.RecruitingOrganizationRecommendationsView +import com.twitter.strato.client.Client + +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class RecommendedRecruitingOrganizationsCandidateSource @Inject() ( + @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client) + extends StratoKeyViewFetcherSource[ + Long, + RecruitingOrganizationRecommendationsView, + RecruitingOrganizationRecommendationsOnUserClientColumn.Value, + Long + ] { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("RecommendedRecruitingOrganizations") + + val fetcher = stratoClient.fetcher[ + Long, + RecruitingOrganizationRecommendationsView, + RecruitingOrganizationRecommendationsOnUserClientColumn.Value + ](RecruitingOrganizationRecommendationsOnUserClientColumn.Path) + + override protected def stratoResultTransformer( + stratoKey: Long, + stratoResult: RecruitingOrganizationRecommendationsOnUserClientColumn.Value + ): Seq[Long] = stratoResult.map(_.apiRecruitingOrganization) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala index d1daeb93e..71279764c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala @@ -1,8 +1,14 @@ package com.twitter.home_mixer.product.for_you.candidate_source import com.google.inject.Provider +import com.twitter.home_mixer.model.HomeFeatures.ServedAuthorIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.SignupCountryFeature +import com.twitter.home_mixer.model.HomeFeatures.SignupSourceFeature import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature +import com.twitter.home_mixer.model.HomeFeatures.UserFollowersCountFeature +import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsForYouRecommendationsFeature +import com.twitter.home_mixer.model.HomeFeatures.ViewerHasPremiumTier import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProductContext @@ -13,8 +19,9 @@ import com.twitter.product_mixer.core.functional_component.candidate_source.prod import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.spam.rtf.{thriftscala => spam} import com.twitter.timelines.render.{thriftscala => tl} -import com.twitter.timelineservice.suggests.{thriftscala => st} import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} import javax.inject.Inject import javax.inject.Singleton @@ -26,7 +33,7 @@ case class ScoredTweetWithConversationMetadata( tweetId: Long, authorId: Long, score: Option[Double] = None, - suggestType: Option[st.SuggestType] = None, + servedType: t.ServedType, sourceTweetId: Option[Long] = None, sourceUserId: Option[Long] = None, quotedTweetId: Option[Long] = None, @@ -37,20 +44,44 @@ case class ScoredTweetWithConversationMetadata( inNetwork: Option[Boolean] = None, sgsValidLikedByUserIds: Option[Seq[Long]] = None, sgsValidFollowedByUserIds: Option[Seq[Long]] = None, + validLikedByUserIds: Option[Seq[Long]] = None, ancestors: Option[Seq[ta.TweetAncestor]] = None, topicId: Option[Long] = None, topicFunctionalityType: Option[tl.TopicContextFunctionalityType] = None, conversationId: Option[Long] = None, conversationFocalTweetId: Option[Long] = None, isReadFromCache: Option[Boolean] = None, - streamToKafka: Option[Boolean] = None, exclusiveConversationAuthorId: Option[Long] = None, authorIsBlueVerified: Option[Boolean] = None, authorIsGoldVerified: Option[Boolean] = None, authorIsGrayVerified: Option[Boolean] = None, authorIsLegacyVerified: Option[Boolean] = None, authorIsCreator: Option[Boolean] = None, - perspectiveFilteredLikedByUserIds: Option[Seq[Long]] = None) + authorFollowers: Option[Long] = None, + locationId: Option[String] = None, + predictionRequestId: Option[Long] = None, + communityId: Option[Long] = None, + communityName: Option[String] = None, + listId: Option[Long] = None, + listName: Option[String] = None, + isNsfw: Option[Boolean] = None, + visibilityReason: Option[spam.FilteredReason] = None, + tweetLanguage: Option[String] = None, + tweetText: Option[String] = None, + tweetTypeMetrics: Option[Seq[Byte]] = None, + debugString: Option[String] = None, + viralContentCreatorFeature: Option[Boolean] = None, + grokContentCreatorFeature: Option[Boolean] = None, + gorkContentCreatorFeature: Option[Boolean] = None, + hasVideoFeature: Option[Boolean] = None, + videoDurationMsFeature: Option[Int] = None, + mediaIds: Option[Seq[Long]] = None, + grokAnnotations: Option[t.GrokAnnotations] = None, + predictedScores: Option[t.PredictedScores] = None, + phoenixPredictedScores: Option[t.PredictedScores] = None, + sourceSignal: Option[t.SourceSignal] = None, + userActionsSize: Option[Int] = None, + userActionsContainsExplicitSignals: Option[Boolean] = None) @Singleton class ScoredTweetsProductCandidateSource @Inject() ( @@ -69,6 +100,18 @@ class ScoredTweetsProductCandidateSource @Inject() ( private val MaxModuleSize = 3 private val MaxAncestorsInConversation = 2 + override def fsCustomMapInput(query: ForYouQuery): Map[String, Int] = { + val userAgeOpt = query.clientContext.userId.map { userId => + SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) + } + val premium = query.features + .map(_.getOrElse(ViewerHasPremiumTier, false)).getOrElse(false) + Map( + "account_age_in_days" -> userAgeOpt.getOrElse(Int.MaxValue), + "premium" -> (if (premium) 1 else 0) + ) + } + override def pipelineRequestTransformer(productPipelineQuery: ForYouQuery): HomeMixerRequest = { HomeMixerRequest( clientContext = productPipelineQuery.clientContext, @@ -78,11 +121,19 @@ class ScoredTweetsProductCandidateSource @Inject() ( productPipelineQuery.deviceContext, productPipelineQuery.seenTweetIds, productPipelineQuery.features.map(_.getOrElse(ServedTweetIdsFeature, Seq.empty)), - productPipelineQuery.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)) - )), + productPipelineQuery.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)), + productPipelineQuery.features.flatMap(_.getOrElse(SignupCountryFeature, None)), + productPipelineQuery.features + .flatMap(_.getOrElse(ViewerAllowsForYouRecommendationsFeature, None)), + productPipelineQuery.features.flatMap(_.getOrElse(SignupSourceFeature, None)), + productPipelineQuery.features.flatMap(_.getOrElse(UserFollowersCountFeature, None)), + productPipelineQuery.features + .map(_.getOrElse(ServedAuthorIdsFeature, Map.empty[Long, Seq[Long]])) + ) + ), serializedRequestCursor = productPipelineQuery.pipelineCursor.map(UrtCursorSerializer.serializeCursor), - maxResults = productPipelineQuery.requestedMaxResults, + maxResults = None, debugParams = None, homeRequestParam = false ) @@ -100,7 +151,7 @@ class ScoredTweetsProductCandidateSource @Inject() ( ScoredTweetWithConversationMetadata( tweetId = ancestor.tweetId, authorId = ancestor.userId, - suggestType = focalTweet.suggestType, + servedType = focalTweet.servedType, conversationId = Some(ancestor.tweetId), conversationFocalTweetId = Some(focalTweet.tweetId), exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId @@ -116,7 +167,7 @@ class ScoredTweetsProductCandidateSource @Inject() ( ScoredTweetWithConversationMetadata( tweetId = ancestor.tweetId, authorId = ancestor.userId, - suggestType = focalTweet.suggestType, + servedType = focalTweet.servedType, inReplyToTweetId = tweetsToParents.get(ancestor).map(_.tweetId), conversationId = conversationId, conversationFocalTweetId = Some(focalTweet.tweetId), @@ -132,7 +183,7 @@ class ScoredTweetsProductCandidateSource @Inject() ( tweetId = focalTweet.tweetId, authorId = focalTweet.authorId, score = focalTweet.score, - suggestType = focalTweet.suggestType, + servedType = focalTweet.servedType, sourceTweetId = focalTweet.sourceTweetId, sourceUserId = focalTweet.sourceUserId, quotedTweetId = focalTweet.quotedTweetId, @@ -143,20 +194,44 @@ class ScoredTweetsProductCandidateSource @Inject() ( inNetwork = focalTweet.inNetwork, sgsValidLikedByUserIds = focalTweet.sgsValidLikedByUserIds, sgsValidFollowedByUserIds = focalTweet.sgsValidFollowedByUserIds, + validLikedByUserIds = focalTweet.validLikedByUserIds, topicId = focalTweet.topicId, topicFunctionalityType = focalTweet.topicFunctionalityType, ancestors = focalTweet.ancestors, conversationId = conversationId, conversationFocalTweetId = conversationFocalTweetId, isReadFromCache = focalTweet.isReadFromCache, - streamToKafka = focalTweet.streamToKafka, exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId, authorIsBlueVerified = focalTweet.authorMetadata.map(_.blueVerified), authorIsGoldVerified = focalTweet.authorMetadata.map(_.goldVerified), authorIsGrayVerified = focalTweet.authorMetadata.map(_.grayVerified), authorIsLegacyVerified = focalTweet.authorMetadata.map(_.legacyVerified), authorIsCreator = focalTweet.authorMetadata.map(_.creator), - perspectiveFilteredLikedByUserIds = focalTweet.perspectiveFilteredLikedByUserIds + authorFollowers = focalTweet.authorMetadata.flatMap(_.followers), + locationId = focalTweet.locationId, + predictionRequestId = focalTweet.predictionRequestId, + communityId = focalTweet.communityId, + communityName = focalTweet.communityName, + listId = focalTweet.listId, + listName = focalTweet.listName, + isNsfw = focalTweet.isNsfw, + visibilityReason = focalTweet.visibilityReason, + tweetLanguage = focalTweet.tweetLanguage, + tweetText = focalTweet.tweetText, + tweetTypeMetrics = focalTweet.tweetTypeMetrics, + debugString = focalTweet.debugString, + viralContentCreatorFeature = focalTweet.viralContentCreator, + hasVideoFeature = focalTweet.hasVideo, + videoDurationMsFeature = focalTweet.videoDurationMs, + mediaIds = focalTweet.mediaIds, + grokAnnotations = focalTweet.grokAnnotations, + predictedScores = focalTweet.predictedScores, + phoenixPredictedScores = focalTweet.phoenixPredictedScores, + sourceSignal = focalTweet.sourceSignal, + userActionsSize = focalTweet.userActionsSize, + userActionsContainsExplicitSignals = focalTweet.userActionsContainsExplicitSignals, + grokContentCreatorFeature = focalTweet.grokContentCreator, + gorkContentCreatorFeature = focalTweet.gorkContentCreator, ) parentScoredTweets :+ focalScoredTweet diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredVideoTweetsCategorizedProductCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredVideoTweetsCategorizedProductCandidateSource.scala new file mode 100644 index 000000000..5373341fa --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredVideoTweetsCategorizedProductCandidateSource.scala @@ -0,0 +1,73 @@ +package com.twitter.home_mixer.product.for_you.candidate_source + +import com.google.inject.Provider +import com.twitter.home_mixer.model.request.HomeMixerRequest +import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct +import com.twitter.home_mixer.model.request.ScoredVideoTweetsProductContext +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.{thriftscala => t} +import com.twitter.product_mixer.core.functional_component.candidate_source.product_pipeline.ProductPipelineCandidateSource +import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry +import com.twitter.snowflake.id.SnowflakeId +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ScoredVideoTweetsCategorizedProductCandidateSource @Inject() ( + override val productPipelineRegistry: Provider[ProductPipelineRegistry], + override val paramsBuilder: Provider[ParamsBuilder]) + extends ProductPipelineCandidateSource[ + ForYouQuery, + HomeMixerRequest, + t.ScoredTweetsResponse, + ScoredVideoTweetCandidate + ] { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("ScoredVideoTweetsCategorizedProduct") + + private val MAX_RESULTS = 20 + + override def fsCustomMapInput(query: ForYouQuery): Map[String, Int] = { + val userAgeOpt = query.clientContext.userId.map { userId => + SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) + } + userAgeOpt.map("account_age_in_days" -> _).toMap + } + + override def pipelineRequestTransformer(productPipelineQuery: ForYouQuery): HomeMixerRequest = { + HomeMixerRequest( + clientContext = productPipelineQuery.clientContext, + product = ScoredVideoTweetsProduct, + productContext = Some( + ScoredVideoTweetsProductContext( + productPipelineQuery.deviceContext, + productPipelineQuery.seenTweetIds, + Some(t.VideoType.Categorized), + None, + None, + None + )), + serializedRequestCursor = None, + maxResults = Some(MAX_RESULTS), + debugParams = None, + homeRequestParam = false + ) + } + + override def productPipelineResultTransformer( + productPipelineResult: t.ScoredTweetsResponse + ): Seq[ScoredVideoTweetCandidate] = { + productPipelineResult.scoredTweets.map { scoredTweet => + ScoredVideoTweetCandidate( + scoredTweet.tweetId, + scoredTweet.authorId, + scoredTweet.score, + scoredTweet.servedType, + scoredTweet.aspectRatio + ) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredVideoTweetsProductCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredVideoTweetsProductCandidateSource.scala new file mode 100644 index 000000000..4cd6d9550 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredVideoTweetsProductCandidateSource.scala @@ -0,0 +1,84 @@ +package com.twitter.home_mixer.product.for_you.candidate_source + +import com.google.inject.Provider +import com.twitter.home_mixer.model.request.HomeMixerRequest +import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct +import com.twitter.home_mixer.model.request.ScoredVideoTweetsProductContext +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.home_mixer.{thriftscala => t} +import com.twitter.product_mixer.core.functional_component.candidate_source.product_pipeline.ProductPipelineCandidateSource +import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry +import com.twitter.snowflake.id.SnowflakeId +import javax.inject.Inject +import javax.inject.Singleton + +/** + * [[ScoredTweetWithConversationMetadata]] + **/ +case class ScoredVideoTweetCandidate( + tweetId: Long, + authorId: Long, + score: Option[Double], + servedType: hmt.ServedType, + aspectRatio: Option[Double]) + +@Singleton +class ScoredVideoTweetsProductCandidateSource @Inject() ( + override val productPipelineRegistry: Provider[ProductPipelineRegistry], + override val paramsBuilder: Provider[ParamsBuilder]) + extends ProductPipelineCandidateSource[ + ForYouQuery, + HomeMixerRequest, + t.ScoredTweetsResponse, + ScoredVideoTweetCandidate + ] { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("ScoredVideoTweetsProduct") + + private val MAX_RESULTS = 20 + + override def fsCustomMapInput(query: ForYouQuery): Map[String, Int] = { + val userAgeOpt = query.clientContext.userId.map { userId => + SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) + } + userAgeOpt.map("account_age_in_days" -> _).toMap + } + + override def pipelineRequestTransformer(productPipelineQuery: ForYouQuery): HomeMixerRequest = { + HomeMixerRequest( + clientContext = productPipelineQuery.clientContext, + product = ScoredVideoTweetsProduct, + productContext = Some( + ScoredVideoTweetsProductContext( + productPipelineQuery.deviceContext, + productPipelineQuery.seenTweetIds, + None, + None, + None, + None + )), + serializedRequestCursor = None, + maxResults = Some(MAX_RESULTS), + debugParams = None, + homeRequestParam = false + ) + } + + override def productPipelineResultTransformer( + productPipelineResult: t.ScoredTweetsResponse + ): Seq[ScoredVideoTweetCandidate] = { + productPipelineResult.scoredTweets.map { scoredTweet => + ScoredVideoTweetCandidate( + scoredTweet.tweetId, + scoredTweet.authorId, + scoredTweet.score, + scoredTweet.servedType, + scoredTweet.aspectRatio + ) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/StoriesModuleCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/StoriesModuleCandidateSource.scala new file mode 100644 index 000000000..d5fade42e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/StoriesModuleCandidateSource.scala @@ -0,0 +1,53 @@ +package com.twitter.home_mixer.product.for_you.candidate_source + +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyFetcherSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.strato.client.Client +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.trendsai.ordered.TrendsModuleOnUserClientColumn +import com.twitter.strato.generated.client.trendsai.ordered.TrendsModuleOnUserClientColumn.Key +import com.twitter.strato.generated.client.trendsai.ordered.TrendsModuleOnUserClientColumn.Value +import com.twitter.strato.graphql.thriftscala.ApiImage +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +case class StoryCandidate( + id: Long, + title: String, + context: String, + hook: Option[String], + thumbnail: Option[ApiImage], + socialProof: Seq[String]) + +@Singleton +class StoriesModuleCandidateSource @Inject() ( + @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client) + extends StratoKeyFetcherSource[ + TrendsModuleOnUserClientColumn.Key, + TrendsModuleOnUserClientColumn.Value, + StoryCandidate + ] { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier(name = "StoriesModule") + + val fetcher: Fetcher[Key, Unit, Value] = stratoClient.fetcher[ + TrendsModuleOnUserClientColumn.Key, + TrendsModuleOnUserClientColumn.View, + TrendsModuleOnUserClientColumn.Value + ](TrendsModuleOnUserClientColumn.Path) + + override protected def stratoResultTransformer(stratoResult: Value): Seq[StoryCandidate] = + stratoResult.items.map { v => + StoryCandidate( + v.id, + v.title, + v.context, + v.hook, + v.thumbnail.map { t => ApiImage(t.url, t.width, t.height) }, + v.facepile + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/TuneFeedCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/TuneFeedCandidateSource.scala new file mode 100644 index 000000000..b3b38e35e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/TuneFeedCandidateSource.scala @@ -0,0 +1,48 @@ +package com.twitter.home_mixer.product.for_you.candidate_source + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.trends.trip.GrokTopicTweetsClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +case class TuneFeedRequest( + grokTopic: Long, + language: Option[String], + placeId: Option[Long]) + +/* + * Takes 3 grok topics the user is following at random, and fetches 1 post for each. + */ +@Singleton +class TuneFeedCandidateSource @Inject() ( + grokTopicTweetsStratoColumn: GrokTopicTweetsClientColumn) + extends CandidateSource[ + TuneFeedRequest, + TweetCandidate + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("TuneFeed") + + private val fetcher = grokTopicTweetsStratoColumn.fetcher + + private val MaxModuleTweets = 20 + + override def apply(tuneFeedRequest: TuneFeedRequest): Stitch[Seq[TweetCandidate]] = { + val key = GrokTopicTweetsClientColumn.Key( + language = tuneFeedRequest.language, + placeId = tuneFeedRequest.placeId, + topicId = tuneFeedRequest.grokTopic + ) + + fetcher.fetch(key, {}).map { response => + response.v + .map(_.map { tweetId => + TweetCandidate(tweetId) + }) + .getOrElse(Seq.empty).take(MaxModuleTweets) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/UnifiedTrendsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/UnifiedTrendsCandidateSource.scala new file mode 100644 index 000000000..797cc4b2c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/UnifiedTrendsCandidateSource.scala @@ -0,0 +1,34 @@ +package com.twitter.home_mixer.product.for_you.candidate_source + +import com.twitter.events.recos.{thriftscala => t} +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +case class TrendCandidate(candidate: t.TrendCandidate, rank: Option[Int]) + +@Singleton +class UnifiedTrendsCandidateSource @Inject() ( + unifiedCandidatesService: t.EventsRecosService.MethodPerEndpoint) + extends CandidateSource[ + t.GetUnfiedCandidatesRequest, + TrendCandidate + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("UnifiedTrends") + + override def apply(request: t.GetUnfiedCandidatesRequest): Stitch[Seq[TrendCandidate]] = { + Stitch.callFuture(unifiedCandidatesService.getUnifiedCandidates(request)).map { + unifiedCandidateResponse => + val trends: Seq[t.TrendCandidate] = unifiedCandidateResponse.candidates.collect { + case t.UnifiedCandidate.Trend(trend) => trend + } + + trends.zipWithIndex.map { + case (trend, index) => TrendCandidate(trend, Some(index + 1)) + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/ArticlePreviewTextFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/ArticlePreviewTextFeatureHydrator.scala new file mode 100644 index 000000000..908b5ad6c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/ArticlePreviewTextFeatureHydrator.scala @@ -0,0 +1,52 @@ +package com.twitter.home_mixer.product.for_you.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.ArticleIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ArticlePreviewTextFeature +import com.twitter.home_mixer.product.for_you.feature_hydrator.ArticlePreviewTextFeatureHydrator.ArticlePreviewTextColumn +import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.strato.client.{Client => StratoClient} +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ArticlePreviewTextFeatureHydrator @Inject() ( + stratoClient: StratoClient) + extends CandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] { + + private val fetcher = + stratoClient.fetcher[Long, Unit, String](ArticlePreviewTextColumn) + + override val features: Set[Feature[_, _]] = + Set(ArticlePreviewTextFeature) + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ArticlePreviewText") + + override def apply( + query: PipelineQuery, + candidate: BaseTweetCandidate, + existingFeatures: FeatureMap + ): Stitch[FeatureMap] = { + + existingFeatures.get(ArticleIdFeature) match { + case Some(articleId) => + fetcher.fetch(articleId).map { fetchResult => + val articlePreviewText: Option[String] = fetchResult.v + FeatureMap(ArticlePreviewTextFeature, articlePreviewText) + } + case None => + Stitch.value(FeatureMap(ArticlePreviewTextFeature, None)) + } + + } +} + +object ArticlePreviewTextFeatureHydrator { + final val ArticlePreviewTextColumn: String = "article/fields/previewText.ArticleEntity" +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel index 87a260b7e..3c7e79a5a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel @@ -5,24 +5,25 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "audience-rewards/thrift/src/main/thrift/common:thrift-scala", + "configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tweetypie", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "stitch/stitch-core", - "stitch/stitch-socialgraph", + "recruiting/candidate-service/src/main/thrift:thrift-scala", + "servo/repo/src/main/scala", "stitch/stitch-timelineservice", "strato/config/columns/audiencerewards/audienceRewardsService:audienceRewardsService-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "util/util-core", + "strato/config/columns/interests:interests-strato-client", + "strato/config/columns/recruiting/api/user:user-strato-client", + "strato/config/columns/socialgraph/service:exists-strato-client", + "strato/config/columns/timelines/pintweet:pintweet-strato-client", + "strato/config/columns/tweetypie/federated:federated-strato-client", + "strato/config/columns/tweetypie/managed:managed-strato-client", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/CurrentPinnedTweetFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/CurrentPinnedTweetFeatureHydrator.scala new file mode 100644 index 000000000..b66c1791e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/CurrentPinnedTweetFeatureHydrator.scala @@ -0,0 +1,54 @@ +package com.twitter.home_mixer.product.for_you.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.CurrentPinnedTweetFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.timelines.pintweet.PinTweetFannedOutUserCacheClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class CurrentPinnedTweetFeatureHydrator @Inject() ( + pinTweetFannedOutUserCacheClientColumn: PinTweetFannedOutUserCacheClientColumn) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("CurrentPinnedTweet") + + override val features: Set[Feature[_, _]] = Set(CurrentPinnedTweetFeature) + + private val fetcher: Fetcher[ + PinTweetFannedOutUserCacheClientColumn.Key, + PinTweetFannedOutUserCacheClientColumn.View, + PinTweetFannedOutUserCacheClientColumn.Value + ] = pinTweetFannedOutUserCacheClientColumn.fetcher + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct + + Stitch + .collectToTry { + authorIds.map { authorId => + fetcher.fetch(authorId).map(_.v.map(data => authorId -> data.tweetId)) + } + }.map { results => + val authorPinTweetMap = results.flatMap(_.toOption).flatten.toMap + candidates.map { candidate => + val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) + val pinnedTweet = authorPinTweetMap.get(authorId) + FeatureMap(CurrentPinnedTweetFeature, pinnedTweet) + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/DisplayedGrokTopicQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/DisplayedGrokTopicQueryFeatureHydrator.scala new file mode 100644 index 000000000..afe31e00b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/DisplayedGrokTopicQueryFeatureHydrator.scala @@ -0,0 +1,37 @@ +package com.twitter.home_mixer.product.for_you.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.CurrentDisplayedGrokTopicFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.interests.FollowedGrokTopicsWithNameOnUserClientColumn +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Random + +@Singleton +class DisplayedGrokTopicQueryFeatureHydrator @Inject() ( + followedGrokTopicsWithNameOnUserClientColumn: FollowedGrokTopicsWithNameOnUserClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "FollowedGrokTopics") + + private val fetcher = followedGrokTopicsWithNameOnUserClientColumn.fetcher + + override def features: Set[Feature[_, _]] = + Set(CurrentDisplayedGrokTopicFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + fetcher + .fetch(query.getRequiredUserId) + .map { result => + val topicResult = result.v.map(_.toSeq).getOrElse(Seq.empty) + val randomTopic = Random.shuffle(topicResult).take(1).headOption + FeatureMap(CurrentDisplayedGrokTopicFeature, randomTopic) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FollowingSportsAccountQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FollowingSportsAccountQueryFeatureHydrator.scala new file mode 100644 index 000000000..b2815bf5f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FollowingSportsAccountQueryFeatureHydrator.scala @@ -0,0 +1,51 @@ +package com.twitter.home_mixer.product.for_you.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.FollowsSportsAccountFeature +import com.twitter.home_mixer.product.for_you.param.ForYouParam.FollowingSportsGateUsersParam +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.socialgraph.{thriftscala => sg} +import com.twitter.strato.generated.client.socialgraph.service.ExistsClientColumn +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class FollowingSportsAccountsQueryFeatureHydrator @Inject() ( + existsClientColumn: ExistsClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("FollowingSportsAccounts") + + override val features: Set[Feature[_, _]] = Set(FollowsSportsAccountFeature) + + val lookupContext = sg.LookupContext(includeInactive = false) + + private val fetcher: Fetcher[ + ExistsClientColumn.Key, + ExistsClientColumn.View, + ExistsClientColumn.Value + ] = existsClientColumn.fetcher + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + query.getOptionalUserId match { + case Some(userId) => + Stitch + .traverse(query.params(FollowingSportsGateUsersParam).toSeq) { sportsAccountId => + val request = + (userId, Seq(sg.Relationship(sg.RelationshipType.Following)), sportsAccountId) + fetcher + .fetch(request, lookupContext).map(_.v.getOrElse(false)) + }.map { followingAccounts: Seq[Boolean] => + FeatureMap(FollowsSportsAccountFeature, followingAccounts.contains(true)) + } + case None => Stitch.value(FeatureMap(FollowsSportsAccountFeature, false)) + } + + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala index e0ae2207e..790d8e4de 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala @@ -1,9 +1,11 @@ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller +import com.twitter.home_mixer.model.HomeFeatures.TLSOriginalTweetsWithAuthorFeature import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.product.for_you.param.ForYouParam.{EnableGetTweetsFromArchiveIndex, EnableTLSHydrationParam} import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -11,6 +13,7 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.stitch.Stitch import com.twitter.stitch.timelineservice.TimelineService import com.twitter.timelineservice.{thriftscala => t} @@ -21,12 +24,29 @@ import javax.inject.Singleton case class TimelineServiceTweetsQueryFeatureHydrator @Inject() ( timelineService: TimelineService, deviceContextMarshaller: DeviceContextMarshaller) - extends QueryFeatureHydrator[PipelineQuery with HasDeviceContext] { + extends QueryFeatureHydrator[PipelineQuery with HasDeviceContext] + with Conditionally[PipelineQuery with HasDeviceContext] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimelineServiceTweets") - override val features: Set[Feature[_, _]] = Set(TimelineServiceTweetsFeature) + override val features: Set[Feature[_, _]] = + Set(TimelineServiceTweetsFeature, TLSOriginalTweetsWithAuthorFeature) + + override def onlyIf(query: PipelineQuery with HasDeviceContext): Boolean = { + query.params(EnableTLSHydrationParam) + } + + private[this] def getTweetsFromArchiveIndexOpt( + query: PipelineQuery with HasDeviceContext + ): Option[Boolean] = { + if (query.params(EnableGetTweetsFromArchiveIndex)) { + // Fall back to the default in timelineservice, i.e. get tweets from archive index. + None + } else { + Some(false) + } + } private val MaxTimelineServiceTweets = 200 @@ -35,7 +55,8 @@ case class TimelineServiceTweetsQueryFeatureHydrator @Inject() ( val timelineQueryOptions = t.TimelineQueryOptions( contextualUserId = query.clientContext.userId, - deviceContext = Some(deviceContextMarshaller(deviceContext, query.clientContext)) + deviceContext = Some(deviceContextMarshaller(deviceContext, query.clientContext)), + getTweetsFromArchiveIndex = getTweetsFromArchiveIndexOpt(query) ) val timelineServiceQuery = t.TimelineQuery( @@ -52,11 +73,22 @@ case class TimelineServiceTweetsQueryFeatureHydrator @Inject() ( case t.TimelineEntry.Tweet(tweet) => tweet.statusId } - FeatureMapBuilder().add(TimelineServiceTweetsFeature, tweets).build() + // Non replies and non retweets + val originalTweetsWithAuthor = timeline.entries.collect { + case t.TimelineEntry.Tweet(tweet) + if tweet.inReplyToStatusId.isEmpty + && tweet.sourceStatusId.isEmpty => + (tweet.statusId, tweet.userId) + } + + FeatureMapBuilder() + .add(TimelineServiceTweetsFeature, tweets) + .add(TLSOriginalTweetsWithAuthorFeature, originalTweetsWithAuthor) + .build() } } override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7) + HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(95) ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetAuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetAuthorFeatureHydrator.scala new file mode 100644 index 000000000..e19c1831a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetAuthorFeatureHydrator.scala @@ -0,0 +1,93 @@ +package com.twitter.home_mixer.product.for_you.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.{ + TLSOriginalTweetsWithAuthorFeature, + TLSOriginalTweetsWithConfirmedAuthorFeature +} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.{FeatureMap, FeatureMapBuilder} +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.spam.rtf.{thriftscala => rtf} +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.tweetypie.managed.HomeMixerTesOnTweetClientColumn +import com.twitter.strato.client.Client +import com.twitter.tweetypie.{thriftscala => tp} +import javax.inject.{Inject, Singleton} +import com.twitter.util.logging.Logging +import javax.inject.Named +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout + +@Singleton +class TweetAuthorFeatureHydrator @Inject() ( + @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client, + stats: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with Logging { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TweetAuthor") + + override val features: Set[Feature[_, _]] = + Set(TLSOriginalTweetsWithConfirmedAuthorFeature) + + private def fetchTweetDataFromTES(tweetId: Long, tweetFieldsOptions: tp.GetTweetFieldsOptions): Stitch[Option[(Long, Long)]] = { + val fetcher = new HomeMixerTesOnTweetClientColumn(stratoClient).fetcher + fetcher + .fetch(tweetId, tweetFieldsOptions) + .map(_.v) + .map { + case Some(result) => + result.tweetResult match { + case tp.TweetFieldsResultState.Found(tweetResult) => + val isRetweet = tweetResult.retweetedTweet.isDefined + tweetResult.tweet.coreData.flatMap { coreData => + val isReply = coreData.reply.isDefined + if (!isRetweet && !isReply) Some(tweetId -> coreData.userId) else None + } + case _ => None + } + case None => None + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val originals: Seq[(Long, Option[Long])] = query.features.map(_.getOrElse(TLSOriginalTweetsWithAuthorFeature, Seq())).getOrElse(Seq.empty) + + if (originals.isEmpty) + return Stitch.value( + FeatureMapBuilder() + .add(TLSOriginalTweetsWithConfirmedAuthorFeature, Seq.empty[(Long, Long)]) + .build() + ) + + val missingIds = originals.collect { case (tid, None) => tid } + + val tweetFieldsOptions = tp.GetTweetFieldsOptions( + tweetIncludes = Set(tp.TweetInclude.TweetFieldId(tp.Tweet.CoreDataField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.CountsField.id)), + safetyLevel = Some(rtf.SafetyLevel.TimelineHome), + forUserId = query.getOptionalUserId + ) + + val fetchedTweetData: Stitch[Map[Long, Long]] = + Stitch + .collect(missingIds.map { id => + fetchTweetDataFromTES(id, tweetFieldsOptions) + }) + .map(_.flatten.toMap) + + fetchedTweetData.map { found => + val rebuilt: Seq[(Long, Long)] = + originals.collect { + case (tid, authorOpt) => + authorOpt.orElse(found.get(tid)).map(aid => (tid, aid)) + }.flatten + + FeatureMapBuilder() + .add(TLSOriginalTweetsWithConfirmedAuthorFeature, rebuilt) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetAuthorFollowersFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetAuthorFollowersFeatureHydrator.scala new file mode 100644 index 000000000..d687689b7 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetAuthorFollowersFeatureHydrator.scala @@ -0,0 +1,69 @@ +package com.twitter.home_mixer.product.for_you.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.gizmoduck.{thriftscala => gt} +import com.twitter.home_mixer.model.HomeFeatures.{ + TLSOriginalTweetsWithConfirmedAuthorFeature, + TweetAuthorFollowersFeature +} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.{FeatureMap, FeatureMapBuilder} +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import javax.inject.{Inject, Singleton} +import com.twitter.util.logging.Logging + +@Singleton +class TweetAuthorFollowersFeatureHydrator @Inject() ( + gizmoduck: gt.UserService.MethodPerEndpoint, + stats: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with Logging { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TweetAuthorFollowers") + + override val features: Set[Feature[_, _]] = + Set(TweetAuthorFollowersFeature) + + private val queryFields: Set[gt.QueryFields] = Set( + gt.QueryFields.Counts + ) + + private val lookupContext = gt.LookupContext(isRequestSheddable = Some(true)) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val originals: Seq[(Long, Long)] = query.features.map(_.getOrElse(TLSOriginalTweetsWithConfirmedAuthorFeature, Seq())).getOrElse(Seq.empty) + + if (originals.isEmpty) { + return Stitch.value( + FeatureMapBuilder() + .add(TweetAuthorFollowersFeature, Map.empty[Long, Option[Long]]) + .build() + ) + } + + val authorIds = originals.map(_._2).distinct + + OffloadFuturePools.offloadFuture { + gizmoduck.get(lookupContext, authorIds, queryFields).map { gizmoduckResponse => + val hydratedUsersMap = gizmoduckResponse.collect { + case userResult if userResult.user.isDefined => + val user = userResult.user.get + user.id -> user + }.toMap.mapValues(user => user.counts.map(_.followers)) + + val tweetAuthorFollowersMap = originals.map { case (tweetId, authorId) => + tweetId -> hydratedUsersMap.getOrElse(authorId, None) + }.toMap + + FeatureMapBuilder() + .add(TweetAuthorFollowersFeature, tweetAuthorFollowersMap) + .build() + } + } + } +} \ No newline at end of file diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetEngagementsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetEngagementsFeatureHydrator.scala new file mode 100644 index 000000000..ee74da1a9 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetEngagementsFeatureHydrator.scala @@ -0,0 +1,91 @@ +package com.twitter.home_mixer.product.for_you.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.{FeatureMap, FeatureMapBuilder} +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.tweetypie.federated.ApiCountsOnTweetClientColumn +import com.twitter.strato.client.Client +import javax.inject.{Inject, Singleton} +import com.twitter.util.logging.Logging +import javax.inject.Named +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout + +case class TweetEngagementCounts( + favoriteCount: Option[Long], + replyCount: Option[Long], + retweetCount: Option[Long], + quoteCount: Option[Long], + bookmarkCount: Option[Long] +) + +object TweetEngagementCountsFeature extends Feature[PipelineQuery, Map[Long, TweetEngagementCounts]] + + +@Singleton +class TweetEngagementsFeatureHydrator @Inject() ( + @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client, + stats: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with Logging { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TweetEngagements") + + override val features: Set[Feature[_, _]] = + Set(TweetEngagementCountsFeature) + + private val engagementCountsFoundCounter = + stats.counter("TweetEngagementsFound") + private val engagementCountsNotFoundCounter = + stats.counter("TweetEngagementsNotFound") + + private def fetchEngagementCountsFromStrato(tweetId: Long): Stitch[Option[TweetEngagementCounts]] = { + val fetcher = new ApiCountsOnTweetClientColumn(stratoClient).fetcher + fetcher + .fetch(tweetId) + .map(_.v) + .map { + case Some(result) => + engagementCountsFoundCounter.incr() + Some(TweetEngagementCounts( + favoriteCount = result.favoriteCount, + replyCount = result.replyCount, + retweetCount = result.retweetCount, + quoteCount = result.quoteCount, + bookmarkCount = result.bookmarkCount + )) + case None => + engagementCountsNotFoundCounter.incr() + None + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val tweetIds: Seq[Long] = query.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty[Long])).getOrElse(Seq.empty) + + if (tweetIds.isEmpty) + return Stitch.value( + FeatureMapBuilder() + .add(TweetEngagementCountsFeature, Map.empty[Long, TweetEngagementCounts]) + .build() + ) + + val fetchedEngagementCounts: Stitch[Map[Long, TweetEngagementCounts]] = + Stitch + .collect(tweetIds.map { id => + fetchEngagementCountsFromStrato(id).map(_.map(counts => id -> counts)) + }) + .map(_.flatten.toMap) + + fetchedEngagementCounts.map { found => + FeatureMapBuilder() + .add(TweetEngagementCountsFeature, found) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala index 07f3ae0e9..84b9f19a0 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala @@ -1,8 +1,12 @@ package com.twitter.home_mixer.product.for_you.feature_hydrator +import com.twitter.home_mixer.model.HomeFeatures.ArticleIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetEntityServiceMigrationParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -13,60 +17,107 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.spam.rtf.{thriftscala => rtf} import com.twitter.stitch.Stitch import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} +import com.twitter.strato.client.Client +import com.twitter.strato.generated.client.tweetypie.managed.HomeMixerOnTweetClientColumn import com.twitter.tweetypie.{thriftscala => TP} +import javax.inject.Named import javax.inject.Inject import javax.inject.Singleton +import com.twitter.finagle.stats.StatsReceiver @Singleton class TweetPreviewTweetypieCandidateFeatureHydrator @Inject() ( - tweetypieStitchClient: TweetypieStitchClient) + statsReceiver: StatsReceiver, + tweetypieStitchClient: TweetypieStitchClient, + @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client) extends CandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] { + private val tweetypieTweetsFoundCounter = + statsReceiver.counter("TweetPreviewTweetypieTweetsFound") + private val tweetypieTweetsNotFoundCounter = + statsReceiver.counter("TweetPreviewTweetypieTweetsNotFound") + private val tesTweetsFoundCounter = + statsReceiver.counter("TweetPreviewTesTweetsFound") + private val tesTweetsNotFoundCounter = + statsReceiver.counter("TweetPreviewTesTweetsNotFound") + private val CoreTweetFields: Set[TP.TweetInclude] = Set[TP.TweetInclude]( TP.TweetInclude.TweetFieldId(TP.Tweet.IdField.id), - TP.TweetInclude.TweetFieldId(TP.Tweet.CoreDataField.id) + TP.TweetInclude.TweetFieldId(TP.Tweet.CoreDataField.id), + TP.TweetInclude.TweetFieldId(TP.Tweet.ArticleField.id), ) private val DefaultFeatureMap = FeatureMapBuilder() .add(TweetTextFeature, None) .add(IsHydratedFeature, false) .add(AuthorIdFeature, None) + .add(VisibilityReason, None) + .add(ArticleIdFeature, None) .build() override val features: Set[Feature[_, _]] = - Set(TweetTextFeature, IsHydratedFeature, AuthorIdFeature) + Set(TweetTextFeature, IsHydratedFeature, AuthorIdFeature, VisibilityReason, ArticleIdFeature) override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetPreviewTweetypie") + private def buildFeatureMap( + gtfResult: Stitch[TP.GetTweetFieldsResult], + isTes: Boolean + ): Stitch[FeatureMap] = { + gtfResult.map { + case TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), _, _) => + if (isTes) tesTweetsFoundCounter.incr() + else tweetypieTweetsFoundCounter.incr() + + val tweetText = found.tweet.coreData.map(_.text) + FeatureMapBuilder(sizeHint = 5) + .add(TweetTextFeature, tweetText) + .add(IsHydratedFeature, true) + .add(AuthorIdFeature, found.tweet.coreData.map(_.userId)) + .add(VisibilityReason, found.suppressReason) + .add(ArticleIdFeature, found.tweet.article.map(_.id)) + .build() + case _ => + if (isTes) tesTweetsNotFoundCounter.incr() + else tweetypieTweetsNotFoundCounter.incr() + DefaultFeatureMap + } + } + override def apply( query: PipelineQuery, candidate: BaseTweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = { - tweetypieStitchClient - .getTweetFields( - tweetId = candidate.id, - options = TP.GetTweetFieldsOptions( - tweetIncludes = CoreTweetFields, - includeRetweetedTweet = false, - includeQuotedTweet = false, - visibilityPolicy = TP.TweetVisibilityPolicy.UserVisible, - safetyLevel = Some(rtf.SafetyLevel.TimelineHomeTweetPreview), - forUserId = query.getOptionalUserId - ) - ).map { - case TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), quoteOpt, _) => - val tweetText = found.tweet.coreData.map(_.text) - FeatureMapBuilder() - .add(TweetTextFeature, tweetText) - .add(IsHydratedFeature, true) - .add(AuthorIdFeature, found.tweet.coreData.map(_.userId)) - .build() - // If no tweet result found, return default features - case _ => - DefaultFeatureMap - } + val getTweetFieldsOptions = TP.GetTweetFieldsOptions( + tweetIncludes = CoreTweetFields, + includeRetweetedTweet = false, + includeQuotedTweet = false, + visibilityPolicy = TP.TweetVisibilityPolicy.UserVisible, + safetyLevel = Some(rtf.SafetyLevel.TimelineHomeTweetPreview), + forUserId = query.getOptionalUserId + ) + if (query.params(EnableTweetEntityServiceMigrationParam)) { + val fetcher = new HomeMixerOnTweetClientColumn(stratoClient).fetcher + fetcher + .fetch( + candidate.id, + getTweetFieldsOptions + ).map(_.v).flatMap { + case Some(result) => buildFeatureMap(Stitch.value(result), true) + case None => + tesTweetsNotFoundCounter.incr() + Stitch.value(DefaultFeatureMap) + } + } else { + val gtfResult: Stitch[TP.GetTweetFieldsResult] = tweetypieStitchClient + .getTweetFields( + tweetId = candidate.id, + options = getTweetFieldsOptions + ) + buildFeatureMap(gtfResult, false) + } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/ViewerHasJobRecommendationsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/ViewerHasJobRecommendationsFeatureHydrator.scala new file mode 100644 index 000000000..6ea07a31b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/ViewerHasJobRecommendationsFeatureHydrator.scala @@ -0,0 +1,82 @@ +package com.twitter.home_mixer.product.for_you.feature_hydrator + +import com.twitter.candidateservice.core_io.MatchingProfile +import com.twitter.home_mixer.model.HomeFeatures.ViewerHasJobRecommendationsEnabled +import com.twitter.home_mixer.model.HomeFeatures.ViewerHasRecruitingOrganizationRecommendationsEnabled +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithDefaultTimeout +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableViewerHasJobRecommendationsFeatureParam +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoErrCategorizer +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Client +import com.twitter.strato.generated.client.recruiting.api.user.MatchingProfileOnUserClientColumn + +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class ViewerHasJobRecommendationsFeatureHydrator @Inject() ( + @Named(BatchedStratoClientWithDefaultTimeout) stratoClient: Client) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ViewerHasJobRecommendations") + + override val features: Set[Feature[_, _]] = + Set(ViewerHasJobRecommendationsEnabled, ViewerHasRecruitingOrganizationRecommendationsEnabled) + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableViewerHasJobRecommendationsFeatureParam) + + private val fetcher = stratoClient.fetcher[ + Long, + Unit, + MatchingProfile + ](MatchingProfileOnUserClientColumn.Path) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + fetcher + .fetch(userId, ()) + .map { result => + val (orgsEnabled, jobsEnabled) = result.v match { + case None => + // When no matching profile exists, the user cannot have consented + // to see job recommendations. Just show org recommendations. + (true, false) + case Some(profile) => + // When a profile does exist, check for consent. Consent cannot be withdrawn. + // It indicates that at some point in the past, the user has opted in to see + // job recommendations. They can later toggle off recommendations. Possible + // states: + // + // 1. User consented and recommendations enabled + // (a) there are recommendations --> show job recommendations + // (b) there are no recommendations --> show org recommendations + // 2. User toggled off recommendations --> show nothing + // 3. User has not consented --> show org recommendations + if (!profile.recommendationsEnabled) { + (false, false) + } else { + val showJobRecommendations = + profile.consentedAt.nonEmpty && profile.hasJobRecommendations + (!showJobRecommendations, showJobRecommendations) + } + } + + FeatureMapBuilder() + .add(ViewerHasRecruitingOrganizationRecommendationsEnabled, orgsEnabled) + .add(ViewerHasJobRecommendationsEnabled, jobsEnabled) + .build() + } + .rescue(StratoErrCategorizer.CategorizeStratoException) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel index 0a875371d..04614da8c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel @@ -5,11 +5,7 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/trends_events", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "stitch/stitch-core", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/NotArticleFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/NotArticleFilter.scala new file mode 100644 index 000000000..daa689e82 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/NotArticleFilter.scala @@ -0,0 +1,29 @@ +package com.twitter.home_mixer.product.for_you.filter + +import com.twitter.home_mixer.model.HomeFeatures.IsArticleFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Filter out tweets that contain articles + */ +object NotArticleFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("NotArticle") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (kept, removed) = candidates.partition { candidate => + !candidate.features.getOrElse(IsArticleFeature, false) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/PromotedTrendFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/PromotedTrendFilter.scala new file mode 100644 index 000000000..27e70df9b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/PromotedTrendFilter.scala @@ -0,0 +1,31 @@ +package com.twitter.home_mixer.product.for_you.filter + +import com.twitter.product_mixer.component_library.model.candidate.trends_events.PromotedTrendNameFeature +import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * A filter that drops promoted trends + */ +object PromotedTrendFilter extends Filter[PipelineQuery, UnifiedTrendCandidate] { + override val identifier = FilterIdentifier("PromotedTrend") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[UnifiedTrendCandidate]] + ): Stitch[FilterResult[UnifiedTrendCandidate]] = { + val filterResult = { + + val (removed, kept) = candidates.partition { candidate => + candidate.features.get(PromotedTrendNameFeature).isDefined + } + FilterResult(kept.map(_.candidate), removed.map(_.candidate)) + } + Stitch.value(filterResult) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala index 61125fcd3..1384c60f1 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala @@ -1,5 +1,6 @@ package com.twitter.home_mixer.product.for_you.filter +import com.twitter.home_mixer.model.HomeFeatures.ArticlePreviewTextFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter @@ -26,7 +27,9 @@ object TweetPreviewTextFilter extends Filter[PipelineQuery, TweetCandidate] { val (kept, removed) = candidates .partition { candidate => - val text = candidate.features.get(TweetTextFeature).getOrElse("") + val text = candidate.features + .get(TweetTextFeature).orElse( + candidate.features.getOrElse(ArticlePreviewTextFeature, None)).getOrElse("") text.length > MinTweetLength && text.take(PreviewTextLength).count(_ == '\n') <= MaxNewlines && diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/BUILD.bazel new file mode 100644 index 000000000..2fb3821db --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/BUILD.bazel @@ -0,0 +1,18 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", + ], + exports = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/FollowingSportsUsersGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/FollowingSportsUsersGate.scala new file mode 100644 index 000000000..8d331becf --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/FollowingSportsUsersGate.scala @@ -0,0 +1,20 @@ +package com.twitter.home_mixer.product.for_you.gate + +import com.twitter.home_mixer.model.HomeFeatures.FollowsSportsAccountFeature +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Continue for all users that follow a sports account + */ +object FollowingSportsUserGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("FollowingSportsUser") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + val followsSportsAccount = query.features.map(_.getOrElse(FollowsSportsAccountFeature, false)) + Stitch.value(followsSportsAccount.contains(true)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/TuneFeedModuleGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/TuneFeedModuleGate.scala new file mode 100644 index 000000000..0325abcab --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/TuneFeedModuleGate.scala @@ -0,0 +1,21 @@ +package com.twitter.home_mixer.product.for_you.gate + +import com.twitter.home_mixer.model.HomeFeatures.CurrentDisplayedGrokTopicFeature +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object TuneFeedModuleGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("TuneFeed") + + override def shouldContinue( + query: PipelineQuery + ): Stitch[Boolean] = { + val grokTopicToDisplay = + query.features.get.getOrElse(CurrentDisplayedGrokTopicFeature, None) + + Stitch.value(grokTopicToDisplay.isDefined) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/UserFollowingRangeGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/UserFollowingRangeGate.scala new file mode 100644 index 000000000..8643d2f72 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/UserFollowingRangeGate.scala @@ -0,0 +1,32 @@ +package com.twitter.home_mixer.product.for_you.gate + +import com.twitter.home_mixer.model.HomeFeatures.UserFollowingCountFeature +import com.twitter.home_mixer.product.for_you.param.ForYouParam.{MinFollowingCountParam, MaxFollowingCountParam} +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Gate that checks if the user's following count is within a valid range. + * The range is controlled by configurable parameters for lower and upper bounds. + */ +object UserFollowingRangeGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("UserFollowingRange") + + override def shouldContinue( + query: PipelineQuery + ): Stitch[Boolean] = { + val userFollowingCount = query.features.get.getOrElse(UserFollowingCountFeature, None) + val lowerBound = query.params(MinFollowingCountParam) + val upperBound = query.params(MaxFollowingCountParam) + + val isInRange = userFollowingCount match { + case Some(count) => count >= lowerBound && count <= upperBound + case None => false // If we don't have the following count, don't continue + } + + Stitch.value(isInRange) + } +} \ No newline at end of file diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.bazel index bcf9519f5..cb14baae4 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.bazel @@ -6,11 +6,7 @@ scala_library( dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala index dda427350..66016b997 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala @@ -26,8 +26,7 @@ case class ForYouQuery( override val features: Option[FeatureMap], override val deviceContext: Option[DeviceContext], override val seenTweetIds: Option[Seq[Long]], - override val dspClientContext: Option[dsp.DspClientContext], - pushToHomeTweetId: Option[Long]) + override val dspClientContext: Option[dsp.DspClientContext]) extends PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.bazel index a56e3a1fd..dae0d7a79 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.bazel @@ -4,11 +4,9 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "util/util-core/src/main/scala/com/twitter/conversions", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/communities_to_join", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala index 5d117199f..4d6c1f01f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala @@ -3,6 +3,9 @@ package com.twitter.home_mixer.product.for_you.param import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.WhoToFollowModuleDisplayType +import com.twitter.product_mixer.component_library.pipeline.candidate.communities_to_join.CommunityToJoinModuleDisplayType +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowUserDisplayType +import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam @@ -13,27 +16,8 @@ import com.twitter.util.Duration object ForYouParam { val SupportedClientFSName = "for_you_supported_client" - - object EnableTopicSocialContextFilterParam - extends FSParam[Boolean]( - name = "for_you_enable_topic_social_context_filter", - default = true - ) - - object EnableVerifiedAuthorSocialContextBypassParam - extends FSParam[Boolean]( - name = "for_you_enable_verified_author_social_context_bypass", - default = true - ) - - object EnableTimelineScorerCandidatePipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_timeline_scorer_candidate_pipeline", - default = false - ) - - object EnableScoredTweetsCandidatePipelineParam - extends BooleanDeciderParam(DeciderKey.EnableForYouScoredTweetsCandidatePipeline) + val StaticParamValueZero = StaticParam(0) + val StaticParamValueFive = StaticParam(5) object EnableWhoToFollowCandidatePipelineParam extends FSParam[Boolean]( @@ -44,27 +28,15 @@ object ForYouParam { object EnableWhoToSubscribeCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_who_to_subscribe", - default = true + default = false ) object EnableTweetPreviewsCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_tweet_previews_candidate_pipeline", - default = true - ) - - object EnablePushToHomeMixerPipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_push_to_home_mixer_pipeline", default = false ) - object EnableScoredTweetsMixerPipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_scored_tweets_mixer_pipeline", - default = true - ) - object ServerMaxResultsParam extends FSBoundedParam[Int]( name = "for_you_server_max_results", @@ -81,14 +53,6 @@ object ForYouParam { max = 100 ) - object WhoToFollowPositionParam - extends FSBoundedParam[Int]( - name = "for_you_who_to_follow_position", - default = 5, - min = 0, - max = 99 - ) - object WhoToFollowMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_who_to_follow_min_injection_interval_in_minutes", @@ -106,20 +70,19 @@ object ForYouParam { enum = WhoToFollowModuleDisplayType ) + object WhoToFollowUserDisplayTypeIdParam + extends FSEnumParam[WhoToFollowUserDisplayType.type]( + name = "for_you_enable_who_to_follow_user_display_type_id", + default = WhoToFollowUserDisplayType.User, + enum = WhoToFollowUserDisplayType + ) + object WhoToFollowDisplayLocationParam extends FSParam[String]( name = "for_you_who_to_follow_display_location", default = "timeline" ) - object WhoToSubscribePositionParam - extends FSBoundedParam[Int]( - name = "for_you_who_to_subscribe_position", - default = 7, - min = 0, - max = 99 - ) - object WhoToSubscribeMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_who_to_subscribe_min_injection_interval_in_minutes", @@ -137,14 +100,6 @@ object ForYouParam { enum = WhoToFollowModuleDisplayType ) - object TweetPreviewsPositionParam - extends FSBoundedParam[Int]( - name = "for_you_tweet_previews_position", - default = 3, - min = 0, - max = 99 - ) - object TweetPreviewsMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_tweet_previews_min_injection_interval_in_minutes", @@ -160,8 +115,6 @@ object ForYouParam { name = "for_you_tweet_previews_max_candidates", default = 1, min = 0, - // NOTE: previews are injected at a fixed position, so max candidates = 1 - // to avoid bunching of previews. max = 1 ) @@ -171,45 +124,579 @@ object ForYouParam { default = true ) - object FlipInlineInjectionModulePosition - extends FSBoundedParam[Int]( - name = "for_you_flip_inline_injection_module_position", - default = 0, - min = 0, - max = 1000 - ) - - object ClearCacheOnPtr { - object EnableParam + object ClearCache { + object PtrEnableParam extends FSParam[Boolean]( name = "for_you_clear_cache_ptr_enable", default = false ) + object ColdStartEnableParam + extends FSParam[Boolean]( + name = "for_you_clear_cache_cold_start_enable", + default = false + ) + + object WarmStartEnableParam + extends FSParam[Boolean]( + name = "for_you_clear_cache_warm_start_enable", + default = false + ) + + object ManualRefreshEnableParam + extends FSParam[Boolean]( + name = "for_you_clear_cache_manual_refresh_enable", + default = false + ) + + object NavigateEnableParam + extends FSParam[Boolean]( + name = "for_you_clear_cache_navigate_enable", + default = false + ) + + object ColdStartRetainViewportParam + extends FSParam[Boolean]( + name = "for_you_clear_cache_retain_viewport", + default = false + ) + case object MinEntriesParam extends FSBoundedParam[Int]( - name = "for_you_clear_cache_ptr_min_entries", + name = "for_you_clear_cache_min_entries", default = 10, min = 0, max = 35 ) } - object EnableClearCacheOnPushToHome + object Navigation { + object PtrEnableParam + extends FSParam[Boolean]( + name = "for_you_navigation_ptr_enable", + default = false + ) + + object ColdStartEnableParam + extends FSParam[Boolean]( + name = "for_you_navigation_cold_start_enable", + default = false + ) + + object WarmStartEnableParam + extends FSParam[Boolean]( + name = "for_you_navigation_warm_start_enable", + default = false + ) + + object ManualRefreshEnableParam + extends FSParam[Boolean]( + name = "for_you_navigation_manual_refresh_enable", + default = false + ) + + object NavigateEnableParam + extends FSParam[Boolean]( + name = "for_you_navigation_navigate_enable", + default = false + ) + } + + /** + * This author ID list is used purely for realtime metrics collection around how often we + * are serving posts from these authors and which sources they are coming from. + */ + object AuthorListForStatsParam + extends FSParam[Set[Long]]( + name = "for_you_author_list_for_stats", + default = Set.empty + ) + + object ExperimentStatsParam + extends FSParam[String]( + name = "for_you_experiment_stats", + default = "" + ) + + object CommunitiesToJoinMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_communities_to_join_min_injection_interval_in_minutes", + default = 2100.minutes, + min = 0.minutes, + max = 6000.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object MaxCommunitiesToJoinCandidatesParam + extends FSBoundedParam[Int]( + name = "for_you_communities_to_join_max_candidates", + default = 3, + min = 1, + max = 10) + + object CommunitiesToJoinDisplayTypeIdParam + extends FSEnumParam[CommunityToJoinModuleDisplayType.type]( + name = "for_you_communities_to_join_display_type_id", + default = CommunityToJoinModuleDisplayType.Carousel, + enum = CommunityToJoinModuleDisplayType + ) + + object EnableCommunitiesToJoinCandidatePipelineParam extends FSParam[Boolean]( - name = "for_you_enable_clear_cache_push_to_home", + name = "for_you_enable_communities_to_join", default = false ) - object EnableServedCandidateKafkaPublishingParam + object RecommendedJobMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_recommended_job_min_injection_interval_in_minutes", + default = 2100.minutes, + min = 0.minutes, + max = 6000.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object MaxRecommendedJobCandidatesParam + extends FSBoundedParam[Int]( + name = "for_you_recommended_job_max_candidates", + default = 3, + min = 1, + max = 10) + + object EnableRecommendedJobsParam extends FSParam[Boolean]( - name = "for_you_enable_served_candidate_kafka_publishing", + name = "for_you_enable_recommended_jobs", + default = false + ) + + object EnableRecommendedRecruitingOrganizationsParam + extends FSParam[Boolean]( + name = "for_you_enable_recommended_recruiting_organizations", + default = false + ) + + object RecommendedRecruitingOrganizationMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_recommended_recruiting_organization_min_injection_interval_in_minutes", + default = 2100.minutes, + min = 0.minutes, + max = 6000.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object MaxRecommendedRecruitingOrganizationCandidatesParam + extends FSBoundedParam[Int]( + name = "for_you_recommended_recruiting_organization_max_candidates", + default = 3, + min = 1, + max = 10 + ) + + object EnableBookmarksCandidatePipelineParam + extends FSParam[Boolean]( + name = "for_you_enable_bookmarks_module", + default = false + ) + + object EnablePinnedTweetsCandidatePipelineParam + extends FSParam[Boolean]( + name = "for_you_enable_pinned_tweets_pipeline", + default = false + ) + + object EnableEntryPointPivotParam + extends FSParam[Boolean]( + name = "for_you_enable_entry_point_pivot", + default = false + ) + + object EnableGrokEntryPointPivotParam + extends FSParam[Boolean]( + name = "for_you_enable_grok_entry_point_pivot", + default = false + ) + + object PinnedTweetsModuleMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_pinned_tweets_module_min_injection_interval_in_minutes", + default = 1440.minutes, // 1 day + min = 0.minutes, + max = 6000.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object EnableExplorationTweetsCandidatePipelineParam + extends FSParam[Boolean]( + name = "for_you_enable_exploration_tweets_pipeline", + default = false + ) + + object EnableJetfuelFramePipelineParam + extends FSParam[Boolean]( + name = "for_you_enable_jetfuel_frame_pipeline", + default = false + ) + + object FollowingSportsGateUsersParam + extends FSParam[Set[Long]]( + name = "for_you_following_sports_users_list", + default = Set.empty + ) + + object ExplorationTweetsTimelinePosition + extends FSBoundedParam[Int]( + name = "for_you_exploration_tweets_position", + default = 15, + min = 0, + max = 50 + ) + + object SuperbowlModuleTimelinePosition + extends FSBoundedParam[Int]( + name = "for_you_superbowl_position", + default = 3, + min = 0, + max = 50 + ) + + object GrokPivotModuleTimelinePosition + extends FSBoundedParam[Int]( + name = "for_you_grok_pivot_position", + default = 3, + min = 0, + max = 50 + ) + + object EntryPointPivotMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_entry_point_pivot_min_injection_interval_in_minutes", + default = 30.minutes, + min = 0.minutes, + max = 6000.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object GrokEntryPointPivotMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_grok_entry_point_pivot_min_injection_interval_in_minutes", + default = 30.minutes, + min = 0.minutes, + max = 6000.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object MaxNumberExplorationTweetsParam + extends FSBoundedParam[Int]( + name = "for_you_exploration_tweets_max_number", + default = 2, + min = 1, + max = 10 + ) + + object EnableBookmarksModuleWeekendGate + extends FSParam[Boolean]( + name = "for_you_enable_bookmarks_module_weekend_gate", + default = false + ) + + object BookmarksModuleMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_bookmarks_module_min_injection_interval_in_minutes", + default = 2880.minutes, // 2 days + min = 0.minutes, + max = 6000.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object InNetworkExplorationTweetsMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + name = "for_you_in_network_exploration_tweets_min_injection_interval_in_minutes", + default = 30.minutes, + min = 0.minutes, + max = 60.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object ExplorationTweetsMaxFollowerCountParam + extends FSBoundedParam[Long]( + name = "for_you_exploration_tweets_max_follower_count", + default = Long.MaxValue, + min = 0L, + max = Long.MaxValue + ) + + object EnableViewerHasJobRecommendationsFeatureParam + extends FSParam[Boolean]( + name = "for_you_enable_viewer_has_job_recommendations_feature", + default = false + ) + + object EnableTrendsParam + extends FSParam[Boolean]( + name = "for_you_enable_trends", + default = false + ) + + object EnableKeywordTrendsParam + extends FSParam[Boolean]( + name = "for_you_enable_keyword_trends", + default = false + ) + + object MaxNumberKeywordTrendsParam + extends FSBoundedParam[Int]( + name = "for_you_keyword_trends_max_number", + default = 5, + min = 1, + max = 10 + ) + + object TrendsModuleMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_trends_min_injection_interval_in_minutes", + default = 360.minutes, + min = 0.minutes, + max = 2880.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object KeywordTrendsModuleMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_keyword_trends_min_injection_interval_in_minutes", + default = 360.minutes, + min = 0.minutes, + max = 2880.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object EnableScoredVideoTweetsCandidatePipelineParam + extends FSParam[Boolean]( + name = "for_you_enable_scored_video_tweets_pipeline", + default = false + ) + + object VideoTweetsModuleMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_video_tweets_module_min_injection_interval_in_minutes", + default = 1.hour, + min = 0.minutes, + max = 6000.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object VideoCarouselNumTweetCandidatesToDedupeAgainstParam + extends FSBoundedParam[Int]( + name = "for_you_video_carousel_num_tweet_candidates_to_dedupe_against", + default = 10, + min = 0, + max = 50 + ) + + object VideoCarouselTimelinePosition + extends FSBoundedParam[Int]( + name = "for_you_video_carousel_position", + default = 5, + min = 0, + max = 50 + ) + + object VideoCarouselNumCandidates + extends FSBoundedParam[Int]( + name = "for_you_video_carousel_num_candidates", + default = 5, + min = 0, + max = 50 + ) + + object VideoCarouselEnableFooterParam + extends FSParam[Boolean]( + name = "for_you_video_carousel_enable_footer", + default = false + ) + + object VideoCarouselAllowVerticalVideos + extends FSParam[Boolean]( + name = "for_you_video_carousel_allow_vertical_videos", default = true ) - object ExperimentStatsParam + object VideoCarouselAllowHorizontalVideos + extends FSParam[Boolean]( + name = "for_you_video_carousel_allow_horizontal_videos", + default = true + ) + + object EnableTLSHydrationParam + extends FSParam[Boolean]( + name = "for_you_enable_tls_hydration", + default = false + ) + + object EnableGetTweetsFromArchiveIndex + extends BooleanDeciderParam(decider = DeciderKey.EnableGetTweetsFromArchiveIndex) + + // Currently only supported by rweb + object EnableArticlePreviewTextHydrationParam + extends FSParam[Boolean]( + name = "for_you_enable_article_preview_hydration", + default = false + ) + + object EnableAdsDebugParam + extends FSParam[Boolean]( + name = "for_you_enable_ads_debug", + default = false + ) + + object RelevancePromptEnableParam + extends FSParam[Boolean]( + name = "for_you_relevance_prompt_enable", + default = false + ) + + object RelevancePromptMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + name = "for_you_relevance_prompt_min_injection_interval_minutes", + default = 1440.minutes, + min = 0.minutes, + max = 144000.minutes + ) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object RelevancePromptTweetPositionParam + extends FSBoundedParam[Int]( + name = "for_you_relevance_prompt_position", + default = 15, + min = 0, + max = 10000 + ) + + object RelevancePromptTitleParam extends FSParam[String]( - name = "for_you_experiment_stats", + name = "for_you_relevance_prompt_title", default = "" ) + + object RelevancePromptPositiveParam + extends FSParam[String]( + name = "for_you_relevance_prompt_positive", + default = "" + ) + + object RelevancePromptNegativeParam + extends FSParam[String]( + name = "for_you_relevance_prompt_negative", + default = "" + ) + + object RelevancePromptNeutralParam + extends FSParam[String]( + name = "for_you_relevance_prompt_neutral", + default = "" + ) + + object EnableForYouTopicSelectorParam + extends FSParam[Boolean]( + name = "for_you_topic_selector_enabled", + default = false + ) + + object ForYouTopicSelectorJetfuelRouteParam + extends FSParam[String]( + name = "for_you_topic_selector_jetfuel_route", + default = "" + ) + + object ForYouTopicSelectorPosition + extends FSBoundedParam[Int]( + name = "for_you_topic_selector_position", + default = 1, + min = 0, + max = 1000 + ) + + object ForYouAppUpsellJetfuelRouteParam + extends FSParam[String]( + name = "for_you_app_upsell_jetfuel_route", + default = "/cards/pivots/appInstallPivot" + ) + + object EnableForYouAppUpsellParam + extends FSParam[Boolean]( + name = "for_you_enable_app_upsell", + default = false + ) + + object ForYouAppUpsellPosition + extends FSBoundedParam[Int]( + name = "for_you_app_upsell_position", + default = 1, + min = 0, + max = 1000 + ) + + object EnableTuneFeedCandidatePipelineParam + extends FSParam[Boolean]( + name = "for_you_enable_tune_feed_pipeline", + default = false + ) + + object TuneFeedTimelinePosition + extends FSBoundedParam[Int]( + name = "for_you_tune_feed_position", + default = 7, + min = 0, + max = 50 + ) + + object TuneFeedModuleMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_tune_feed_module_min_injection_interval_in_minutes", + default = 180.minutes, // 3 hours + min = 0.minutes, + max = 6000.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object EnableFollowedGrokTopicsHydrationParam + extends FSParam[Boolean]( + name = "for_you_enable_followed_grok_topics_hydration", + default = false + ) + + object EnableForYouTimelineAdsSurface + extends FSParam[Boolean]( + name = "for_you_enable_timeline_ads_surface", + default = false + ) + + object MinFollowingCountParam + extends FSBoundedParam[Int]( + name = "for_you_user_following_range_gate_min_following_count", + default = 0, + min = 0, + max = 1000000 + ) + + object MaxFollowingCountParam + extends FSBoundedParam[Int]( + name = "for_you_user_following_range_gate_max_following_count", + default = 10000, + min = 0, + max = 1000000 + ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala index 001ee57ad..727b4c960 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala @@ -12,49 +12,119 @@ class ForYouParamConfig @Inject() () extends ProductParamConfig { override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableForYouProduct override val supportedClientFSName: String = SupportedClientFSName - override val booleanDeciderOverrides = Seq( - EnableScoredTweetsCandidatePipelineParam - ) - override val booleanFSOverrides = Seq( - ClearCacheOnPtr.EnableParam, + ClearCache.PtrEnableParam, + ClearCache.ColdStartEnableParam, + ClearCache.WarmStartEnableParam, + ClearCache.ManualRefreshEnableParam, + ClearCache.NavigateEnableParam, + ClearCache.ColdStartRetainViewportParam, + EnableCommunitiesToJoinCandidatePipelineParam, + EnableBookmarksCandidatePipelineParam, + EnableExplorationTweetsCandidatePipelineParam, + EnableJetfuelFramePipelineParam, + EnableBookmarksModuleWeekendGate, + EnablePinnedTweetsCandidatePipelineParam, + EnableEntryPointPivotParam, EnableFlipInjectionModuleCandidatePipelineParam, - EnablePushToHomeMixerPipelineParam, - EnableScoredTweetsMixerPipelineParam, - EnableServedCandidateKafkaPublishingParam, - EnableTimelineScorerCandidatePipelineParam, - EnableTopicSocialContextFilterParam, - EnableVerifiedAuthorSocialContextBypassParam, + EnableGrokEntryPointPivotParam, + EnableRecommendedJobsParam, + EnableRecommendedRecruitingOrganizationsParam, + EnableTweetPreviewsCandidatePipelineParam, + EnableViewerHasJobRecommendationsFeatureParam, EnableWhoToFollowCandidatePipelineParam, EnableWhoToSubscribeCandidatePipelineParam, - EnableTweetPreviewsCandidatePipelineParam, - EnableClearCacheOnPushToHome + EnableTrendsParam, + EnableKeywordTrendsParam, + EnableArticlePreviewTextHydrationParam, + EnableForYouTimelineAdsSurface, + EnableScoredVideoTweetsCandidatePipelineParam, + VideoCarouselEnableFooterParam, + VideoCarouselAllowVerticalVideos, + VideoCarouselAllowHorizontalVideos, + RelevancePromptEnableParam, + Navigation.PtrEnableParam, + Navigation.ColdStartEnableParam, + Navigation.WarmStartEnableParam, + Navigation.ManualRefreshEnableParam, + Navigation.NavigateEnableParam, + EnableAdsDebugParam, + EnableForYouAppUpsellParam, + EnableForYouTopicSelectorParam, + EnableTuneFeedCandidatePipelineParam, + EnableFollowedGrokTopicsHydrationParam, + EnableTLSHydrationParam + ) + + override val boundedLongFSOverrides = Seq( + ExplorationTweetsMaxFollowerCountParam ) override val boundedIntFSOverrides = Seq( AdsNumOrganicItemsParam, - ClearCacheOnPtr.MinEntriesParam, - FlipInlineInjectionModulePosition, + ClearCache.MinEntriesParam, + ExplorationTweetsTimelinePosition, + GrokPivotModuleTimelinePosition, + SuperbowlModuleTimelinePosition, + VideoCarouselNumTweetCandidatesToDedupeAgainstParam, + VideoCarouselTimelinePosition, + VideoCarouselNumCandidates, + MaxCommunitiesToJoinCandidatesParam, + MaxNumberExplorationTweetsParam, + MaxNumberKeywordTrendsParam, + MaxRecommendedJobCandidatesParam, + MaxRecommendedRecruitingOrganizationCandidatesParam, ServerMaxResultsParam, - WhoToFollowPositionParam, - WhoToSubscribePositionParam, - TweetPreviewsPositionParam, - TweetPreviewsMaxCandidatesParam + TweetPreviewsMaxCandidatesParam, + RelevancePromptTweetPositionParam, + ForYouAppUpsellPosition, + ForYouTopicSelectorPosition, + TuneFeedTimelinePosition, + MinFollowingCountParam, + MaxFollowingCountParam ) override val stringFSOverrides = Seq( WhoToFollowDisplayLocationParam, - ExperimentStatsParam + ExperimentStatsParam, + RelevancePromptTitleParam, + RelevancePromptPositiveParam, + RelevancePromptNegativeParam, + RelevancePromptNeutralParam, + ForYouAppUpsellJetfuelRouteParam, + ForYouTopicSelectorJetfuelRouteParam, ) override val boundedDurationFSOverrides = Seq( + CommunitiesToJoinMinInjectionIntervalParam, + RecommendedJobMinInjectionIntervalParam, + RecommendedRecruitingOrganizationMinInjectionIntervalParam, WhoToFollowMinInjectionIntervalParam, WhoToSubscribeMinInjectionIntervalParam, - TweetPreviewsMinInjectionIntervalParam + TweetPreviewsMinInjectionIntervalParam, + BookmarksModuleMinInjectionIntervalParam, + InNetworkExplorationTweetsMinInjectionIntervalParam, + PinnedTweetsModuleMinInjectionIntervalParam, + TrendsModuleMinInjectionIntervalParam, + KeywordTrendsModuleMinInjectionIntervalParam, + EntryPointPivotMinInjectionIntervalParam, + GrokEntryPointPivotMinInjectionIntervalParam, + VideoTweetsModuleMinInjectionIntervalParam, + RelevancePromptMinInjectionIntervalParam ) override val enumFSOverrides = Seq( WhoToFollowDisplayTypeIdParam, - WhoToSubscribeDisplayTypeIdParam + WhoToSubscribeDisplayTypeIdParam, + WhoToFollowUserDisplayTypeIdParam, + CommunitiesToJoinDisplayTypeIdParam + ) + + override val booleanDeciderOverrides = + Seq(EnableGetTweetsFromArchiveIndex) + + override val longSetFSOverrides = Seq( + AuthorListForStatsParam, + FollowingSportsGateUsersParam ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel index 17e99c46c..c6d917685 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel @@ -4,13 +4,12 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", + "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "src/java/com/twitter/search/common/schema/earlybird", "src/java/com/twitter/search/queryparser/query:core-query-nodes", "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/UnifiedCandidatesQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/UnifiedCandidatesQueryTransformer.scala new file mode 100644 index 000000000..c942dd155 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/UnifiedCandidatesQueryTransformer.scala @@ -0,0 +1,34 @@ +package com.twitter.home_mixer.product.for_you.query_transformer + +import com.twitter.events.recos.{thriftscala => t} +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.Param + +case class UnifiedCandidatesQueryTransformer( + maxResultsParam: Param[Int], + candidatePipelineIdentifier: CandidatePipelineIdentifier) + extends CandidatePipelineQueryTransformer[ + PipelineQuery, + t.GetUnfiedCandidatesRequest + ] { + override val identifier: TransformerIdentifier = TransformerIdentifier("UnifiedCandidates") + + override def transform( + query: PipelineQuery + ): t.GetUnfiedCandidatesRequest = { + + t.GetUnfiedCandidatesRequest( + displayLocation = t.DisplayLocation.Guide, + clientId = query.clientContext.appId, + userId = query.getOptionalUserId, + languageCode = query.getLanguageCode, + countryCode = query.getCountryCode, + maxResults = Some(query.params(maxResultsParam)), + userAgent = query.clientContext.userAgent, + isObjectiveTrendsRequest = Some(true) + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel index cf629b665..dfccf8b0d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel @@ -4,10 +4,14 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", + "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/trends_events", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "src/java/com/twitter/search/common/schema/earlybird", + "src/thrift/com/twitter/frigate/bookmarks:bookmarks-thrift-scala", "src/thrift/com/twitter/search:earlybird-scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BookmarksResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BookmarksResponseFeatureTransformer.scala new file mode 100644 index 000000000..9c1007a6a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BookmarksResponseFeatureTransformer.scala @@ -0,0 +1,25 @@ +package com.twitter.home_mixer.product.for_you.response_transformer + +import com.twitter.frigate.bookmarks.thriftscala.BookmarkedTweet +import com.twitter.home_mixer.model.HomeFeatures.BookmarkedTweetTimestamp +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier + +object BookmarksResponseFeatureTransformer extends CandidateFeatureTransformer[BookmarkedTweet] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("BookmarksResponse") + + override val features: Set[Feature[_, _]] = + Set(BookmarkedTweetTimestamp, ServedTypeFeature) + + def transform(input: BookmarkedTweet): FeatureMap = FeatureMapBuilder() + .add(BookmarkedTweetTimestamp, Some(input.timestamp)) + .add(ServedTypeFeature, hmt.ServedType.ForYouResurfacedBookmark) + .build() +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/ExplorationTweetResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/ExplorationTweetResponseFeatureTransformer.scala new file mode 100644 index 000000000..75ac1c347 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/ExplorationTweetResponseFeatureTransformer.scala @@ -0,0 +1,26 @@ +package com.twitter.home_mixer.product.for_you.response_transformer + +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier + +object ExplorationTweetResponseFeatureTransformer extends CandidateFeatureTransformer[Long] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ExplorationTweetResponse") + + override val features: Set[Feature[_, _]] = + Set(ServedTypeFeature) + + def transform( + input: Long + ): FeatureMap = { + FeatureMapBuilder() + .add(ServedTypeFeature, hmt.ServedType.ForYouExploration) + .build() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/KeywordTrendsFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/KeywordTrendsFeatureTransformer.scala new file mode 100644 index 000000000..2bd29f311 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/KeywordTrendsFeatureTransformer.scala @@ -0,0 +1,53 @@ +package com.twitter.home_mixer.product.for_you.response_transformer + +import com.twitter.events.recos.{thriftscala => t} +import com.twitter.home_mixer.util.UrtUtil +import com.twitter.home_mixer.product.for_you.candidate_source.TrendCandidate +import com.twitter.product_mixer.component_library.model.candidate.trends_events._ +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.trend.GroupedTrend + +object KeywordTrendsFeatureTransformer extends CandidateFeatureTransformer[TrendCandidate] { + override val identifier: TransformerIdentifier = TransformerIdentifier("KeywordTrends") + + override def features: Set[Feature[_, _]] = Set( + TrendNormalizedNameFeature, + TrendNameFeature, + TrendUrlFeature, + TrendDescriptionFeature, + TrendTweetCountFeature, + TrendDomainContextFeature, + TrendGroupedTrendsFeature, + PromotedTrendNameFeature, + TrendRankFeature + ) + + override def transform(input: TrendCandidate): FeatureMap = { + val trend: t.TrendCandidate = input.candidate + + FeatureMapBuilder() + .add(TrendNameFeature, trend.trendName) + .add(TrendDescriptionFeature, trend.context.flatMap(_.description)) + .add(TrendNormalizedNameFeature, trend.normalizedTrendName) + .add(TrendUrlFeature, UrtUtil.transformUrl(trend.url)) + .add(TrendTweetCountFeature, trend.context.flatMap(_.tweetCount)) + .add(TrendDomainContextFeature, trend.domainContext) + .add(TrendGroupedTrendsFeature, trend.relatedTrends.map(transformGroupedTrends)) + .add(PromotedTrendNameFeature, trend.promotedMetadata.map(_.name)) + .add(TrendRankFeature, input.rank) + .build() + } + + private def transformGroupedTrends(groupedTrends: Seq[t.RelatedTrend]): Seq[GroupedTrend] = { + groupedTrends.map { trend => + GroupedTrend( + trendName = trend.trendName, + url = UrtUtil.transformUrl(trend.url) + ) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/PinnedTweetResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/PinnedTweetResponseFeatureTransformer.scala new file mode 100644 index 000000000..fbcd36555 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/PinnedTweetResponseFeatureTransformer.scala @@ -0,0 +1,30 @@ +package com.twitter.home_mixer.product.for_you.response_transformer + +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.product.for_you.candidate_source.PinnedTweetCandidate +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier + +object PinnedTweetResponseFeatureTransformer + extends CandidateFeatureTransformer[PinnedTweetCandidate] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("PinnedTweetResponse") + + override val features: Set[Feature[_, _]] = + Set(AuthorIdFeature, ServedTypeFeature) + + def transform( + input: PinnedTweetCandidate + ): FeatureMap = { + FeatureMapBuilder() + .add(AuthorIdFeature, input.userId) + .add(ServedTypeFeature, hmt.ServedType.ForYouPinned) + .build() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/ScoredVideoTweetResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/ScoredVideoTweetResponseFeatureTransformer.scala new file mode 100644 index 000000000..c0d037ab1 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/ScoredVideoTweetResponseFeatureTransformer.scala @@ -0,0 +1,38 @@ +package com.twitter.home_mixer.product.for_you.response_transformer + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoAspectRatioFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoDisplayTypeFeature +import com.twitter.home_mixer.product.for_you.candidate_source.ScoredVideoTweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Carousel +import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.CompactCarousel + +object ScoredVideoTweetResponseFeatureTransformer + extends CandidateFeatureTransformer[ScoredVideoTweetCandidate] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredVideoTweetResponse") + + override val features: Set[Feature[_, _]] = + Set(AuthorIdFeature, ServedTypeFeature, VideoAspectRatioFeature, VideoDisplayTypeFeature) + + def transform( + input: ScoredVideoTweetCandidate + ): FeatureMap = { + val displayType = input.aspectRatio.map { ratio => + if (ratio > 1.0) Carousel else CompactCarousel + } + FeatureMapBuilder() + .add(AuthorIdFeature, Some(input.authorId)) + .add(ServedTypeFeature, input.servedType) + .add(VideoAspectRatioFeature, input.aspectRatio.map(_.toFloat)) + .add(VideoDisplayTypeFeature, displayType) + .build() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/StoriesModuleResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/StoriesModuleResponseFeatureTransformer.scala new file mode 100644 index 000000000..78d1261a5 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/StoriesModuleResponseFeatureTransformer.scala @@ -0,0 +1,51 @@ +package com.twitter.home_mixer.product.for_you.response_transformer + +import com.twitter.home_mixer.product.for_you.candidate_source.StoryCandidate +import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendDescriptionFeature +import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendDomainContextFeature +import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendNameFeature +import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendSocialContextImages +import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendThumbnail +import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendUrlFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.urt.graphql.ApiImage +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DeepLink +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url + +object StoriesModuleResponseFeatureTransformer extends CandidateFeatureTransformer[StoryCandidate] { + + override val identifier: TransformerIdentifier = TransformerIdentifier("StoriesModuleResponse") + + override val features: Set[Feature[_, _]] = Set( + TrendNameFeature, + TrendDescriptionFeature, + TrendUrlFeature, + TrendDomainContextFeature, + TrendSocialContextImages, + TrendThumbnail + ) + + def transform(input: StoryCandidate): FeatureMap = { + val url = input.id + val thumbnail = input.thumbnail.map { img => + ApiImage( + height = img.originalImgHeight, + width = img.originalImgWidth, + url = img.originalImgUrl + ) + } + + FeatureMapBuilder() + .add(TrendNameFeature, input.title) + .add(TrendDescriptionFeature, None) + .add(TrendUrlFeature, Url(DeepLink, url)) + .add(TrendDomainContextFeature, Some(input.context)) + .add(TrendSocialContextImages, Some(input.socialProof)) + .add(TrendThumbnail, thumbnail) + .build() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TuneFeedFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TuneFeedFeatureTransformer.scala new file mode 100644 index 000000000..19527d9d4 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TuneFeedFeatureTransformer.scala @@ -0,0 +1,20 @@ +package com.twitter.home_mixer.product.for_you.response_transformer + +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier + +object TuneFeedFeatureTransformer extends CandidateFeatureTransformer[TweetCandidate] { + + override val identifier: TransformerIdentifier = TransformerIdentifier("TuneFeed") + + override val features: Set[Feature[_, _]] = Set(ServedTypeFeature) + + def transform( + input: TweetCandidate + ): FeatureMap = FeatureMap(ServedTypeFeature, hmt.ServedType.ForYouPopularTopic) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala index 8b783db38..a0cc73e30 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala @@ -1,14 +1,14 @@ package com.twitter.home_mixer.product.for_you.response_transformer +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.{thriftscala => st} import com.twitter.search.earlybird.{thriftscala => eb} object TweetPreviewResponseFeatureTransformer @@ -18,14 +18,14 @@ object TweetPreviewResponseFeatureTransformer TransformerIdentifier("TweetPreviewResponse") override val features: Set[Feature[_, _]] = - Set(AuthorIdFeature, IsTweetPreviewFeature, SuggestTypeFeature) + Set(AuthorIdFeature, IsTweetPreviewFeature, ServedTypeFeature) def transform( input: eb.ThriftSearchResult ): FeatureMap = { FeatureMapBuilder() .add(IsTweetPreviewFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.TweetPreview)) + .add(ServedTypeFeature, hmt.ServedType.ForYouTweetPreview) .add(AuthorIdFeature, input.metadata.map(_.fromUserId)) .build() } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel index c320c8838..a07b339ce 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel @@ -4,7 +4,5 @@ scala_library( strict_deps = True, dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "timelineservice/common:model", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/PinnedTweetCandidateScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/PinnedTweetCandidateScorer.scala new file mode 100644 index 000000000..dcceb67d3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/PinnedTweetCandidateScorer.scala @@ -0,0 +1,31 @@ +package com.twitter.home_mixer.product.for_you.scorer + +import com.twitter.home_mixer.model.HomeFeatures.AuthorFollowersFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object PinnedTweetCandidateScorer extends Scorer[PipelineQuery, TweetCandidate] { + override val identifier: ScorerIdentifier = ScorerIdentifier("PinnedTweetCandidate") + + override def features: Set[Feature[_, _]] = Set(ScoreFeature) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + + val featureMaps = candidates.map { candidate => + val score = candidate.features.getOrElse(AuthorFollowersFeature, None) + FeatureMap(ScoreFeature, score.map(_.toDouble)) + } + + Stitch.value(featureMaps) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/BUILD.bazel new file mode 100644 index 000000000..fec8e133f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/BUILD.bazel @@ -0,0 +1,11 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/DebugUpdateSortAdsResult.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/DebugUpdateSortAdsResult.scala new file mode 100644 index 000000000..93d2c66cf --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/DebugUpdateSortAdsResult.scala @@ -0,0 +1,31 @@ +package com.twitter.home_mixer.product.for_you.selector + +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates +import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +// This selector is used for debugging Ads. Tt will put all the Ads candidates to the top +case class DebugUpdateSortAdsResult( + adsCandidatePipeline: CandidatePipelineIdentifier) + extends Selector[PipelineQuery] { + + override val pipelineScope: CandidateScope = SpecificPipeline(adsCandidatePipeline) + + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val PartitionedCandidates(adCandidates, otherRemainingCandidates) = + pipelineScope.partition(result) + + SelectorResult( + remainingCandidates = remainingCandidates, + result = adCandidates ++ otherRemainingCandidates) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/DebunchCandidates.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/DebunchCandidates.scala new file mode 100644 index 000000000..3fa913372 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/DebunchCandidates.scala @@ -0,0 +1,70 @@ +package com.twitter.home_mixer.product.for_you.selector + +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.Param + +trait MustDebunch { + def apply(candidate: CandidateWithDetails): Boolean +} + +/** + * This selector rearranges the candidates to only allow bunches of size [[maxBunchSize]], + * where a bunch is a consecutive sequence of candidates that meet [[mustDebunch]]. + */ +case class DebunchCandidates( + override val pipelineScope: CandidateScope, + mustDebunch: MustDebunch, + maxBunchSize: Param[Int]) + extends Selector[PipelineQuery] { + + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val PartitionedCandidates(selectedCandidates, otherCandidates) = + pipelineScope.partition(remainingCandidates) + val mutableCandidates = collection.mutable.ListBuffer(selectedCandidates: _*) + + var candidatePointer = 0 + var nonDebunchPointer = 0 + var bunchSize = 0 + var finalNonDebunch = -1 + + while (candidatePointer < mutableCandidates.size) { + if (mustDebunch(mutableCandidates(candidatePointer))) bunchSize += 1 + else { + bunchSize = 0 + finalNonDebunch = candidatePointer + } + + if (bunchSize > query.params(maxBunchSize)) { + nonDebunchPointer = Math.max(candidatePointer, nonDebunchPointer) + while (nonDebunchPointer < mutableCandidates.size && + mustDebunch(mutableCandidates(nonDebunchPointer))) { + nonDebunchPointer += 1 + } + if (nonDebunchPointer == mutableCandidates.size) + candidatePointer = mutableCandidates.size + else { + val nextNonDebunch = mutableCandidates(nonDebunchPointer) + mutableCandidates.remove(nonDebunchPointer) + mutableCandidates.insert(candidatePointer, nextNonDebunch) + bunchSize = 0 + finalNonDebunch = candidatePointer + } + } + + candidatePointer += 1 + } + + val debunchedCandidates = mutableCandidates.toList + val updatedCandidates = otherCandidates ++ debunchedCandidates + SelectorResult(remainingCandidates = updatedCandidates, result = result) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/RemoveDuplicateCandidatesOutsideModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/RemoveDuplicateCandidatesOutsideModule.scala new file mode 100644 index 000000000..aeddb7c56 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/RemoveDuplicateCandidatesOutsideModule.scala @@ -0,0 +1,46 @@ +package com.twitter.home_mixer.product.for_you.selector + +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails +import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.Param + +case class RemoveDuplicateCandidatesOutsideModule( + override val pipelineScope: CandidateScope, + candidatePipelinesOutsideModule: Set[CandidatePipelineIdentifier], + numCandidatesToCompareAgainst: Param[Int]) + extends Selector[PipelineQuery] { + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val candidatesOutsideModule = getCandidateIdsOutsideModule(query, remainingCandidates) + val finalRemainingCandidates = remainingCandidates.map { + case module: ModuleCandidateWithDetails if pipelineScope.contains(module) => + module.copy(candidates = module.candidates.filterNot { candidate => + candidatesOutsideModule.contains(candidate.candidateIdLong) + }) + case candidate => candidate + } + + SelectorResult(remainingCandidates = finalRemainingCandidates, result = result) + } + + private def getCandidateIdsOutsideModule( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails] + ): Set[Long] = { + remainingCandidates + .collect { + case candidate: ItemCandidateWithDetails + if candidatePipelinesOutsideModule.contains(candidate.source) => + candidate.candidateIdLong + }.take(query.params(numCandidatesToCompareAgainst)).toSet + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel index 90faf3d45..b30ca64b0 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel @@ -9,16 +9,14 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "home-mixer/thrift/src/main/thrift:thrift-scala", "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/features/common", "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", "timelines/ml:kafka", - "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka", "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala index bfa7dd52e..18895dcac 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala @@ -1,16 +1,13 @@ package com.twitter.home_mixer.product.for_you.side_effect -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableServedCandidateKafkaPublishingParam +import com.twitter.home_mixer.param.HomeGlobalParams.EnableServedCandidateFeatureKeysKafkaPublishingParam import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier @@ -23,7 +20,6 @@ import com.twitter.timelines.ml.kafka.serde.CandidateFeatureKeySerde import com.twitter.timelines.ml.kafka.serde.TBaseSerde import com.twitter.timelines.served_candidates_logging.{thriftscala => sc} import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} -import com.twitter.timelineservice.suggests.{thriftscala => tls} import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.common.serialization.Serializer import scala.collection.JavaConverters._ @@ -42,8 +38,6 @@ class ServedCandidateFeatureKeysKafkaSideEffect( ] with PipelineResultSideEffect.Conditionally[PipelineQuery, Timeline] { - import ServedCandidateKafkaSideEffect._ - override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedCandidateFeatureKeys") override def onlyIf( @@ -52,7 +46,7 @@ class ServedCandidateFeatureKeysKafkaSideEffect( remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: Timeline - ): Boolean = query.params.getBoolean(EnableServedCandidateKafkaPublishingParam) + ): Boolean = query.params.getBoolean(EnableServedCandidateFeatureKeysKafkaPublishingParam) override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" @@ -71,39 +65,42 @@ class ServedCandidateFeatureKeysKafkaSideEffect( droppedCandidates: Seq[CandidateWithDetails], response: Timeline ): Seq[ProducerRecord[sc.CandidateFeatureKey, pldr.PolyDataRecord]] = { - val servedRequestIdOpt = - query.features.getOrElse(FeatureMap.empty).getOrElse(ServedRequestIdFeature, None) - - extractCandidates(query, selectedCandidates, sourceIdentifiers).map { candidate => - val isReadFromCache = candidate.features.getOrElse(IsReadFromCacheFeature, false) - val servedId = candidate.features.get(ServedIdFeature).get - - val key = sc.CandidateFeatureKey( - tweetId = candidate.candidateIdLong, - viewerId = query.getRequiredUserId, - servedId = servedId) + query.features + .flatMap(_.getOrElse(ServedIdFeature, None)) + .fold(Seq.empty[ProducerRecord[sc.CandidateFeatureKey, pldr.PolyDataRecord]]) { servedId => + CandidatesUtil + .getItemCandidates { + selectedCandidates.iterator + .filter(candidate => sourceIdentifiers.contains(candidate.source)).toSeq + } + .flatMap { candidate => + candidate.features + .getOrElse(PredictionRequestIdFeature, None) + .map { predictionRequestId => + val key = sc.CandidateFeatureKey( + tweetId = candidate.candidateIdLong, + viewerId = query.getRequiredUserId, + servedId = -1L) - val record = - ServedCandidateFeatureKeysAdapter - .adaptToDataRecords( - ServedCandidateFeatureKeysFields( - candidateTweetSourceId = candidate.features - .getOrElse(CandidateSourceIdFeature, None).map(_.value.toLong).getOrElse(2L), - predictionRequestId = - candidate.features.getOrElse(PredictionRequestIdFeature, None).get, - servedRequestIdOpt = if (isReadFromCache) servedRequestIdOpt else None, - servedId = servedId, - injectionModuleName = candidate.getClass.getSimpleName, - viewerFollowsOriginalAuthor = - Some(candidate.features.getOrElse(InNetworkFeature, true)), - suggestType = candidate.features - .getOrElse(SuggestTypeFeature, None).getOrElse(tls.SuggestType.RankedOrganicTweet), - finalPositionIndex = Some(candidate.sourcePosition), - isReadFromCache = isReadFromCache - )).asScala.head + val record = + ServedCandidateFeatureKeysAdapter + .adaptToDataRecords(ServedCandidateFeatureKeysFields( + viewerId = query.getRequiredUserId, + tweetId = candidate.candidateIdLong, + predictionRequestId = predictionRequestId, + servedRequestIdOpt = None, + servedId = servedId, + injectionModuleName = candidate.getClass.getSimpleName, + viewerFollowsOriginalAuthor = + Some(candidate.features.getOrElse(InNetworkFeature, true)), + finalPositionIndex = Some(candidate.sourcePosition), + isReadFromCache = candidate.features.getOrElse(IsReadFromCacheFeature, false) + )).asScala.head - new ProducerRecord(topic, key, pldr.PolyDataRecord.dataRecord(record)) - } + new ProducerRecord(topic, key, pldr.PolyDataRecord.dataRecord(record)) + } + } + } } override val alerts = Seq( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala index be7c2a533..966f2cb0b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala @@ -1,30 +1,40 @@ package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ExperimentStatsParam +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.AuthorListForDataCollectionParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.AuthorListForStatsParam +import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton +import com.twitter.util.logging.Logging -@Singleton -class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver) - extends PipelineResultSideEffect[PipelineQuery, Timeline] { +case class ServedStatsSideEffect( + candidatePipelines: Set[CandidatePipelineIdentifier], + statsReceiver: StatsReceiver) + extends PipelineResultSideEffect[PipelineQuery, Timeline] + with Logging { override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats") private val baseStatsReceiver = statsReceiver.scope(identifier.toString) - private val suggestTypeStatsReceiver = baseStatsReceiver.scope("SuggestType") + private val authorStatsReceiver = baseStatsReceiver.scope("Author") + private val tweetStatsReceiver = baseStatsReceiver.scope("Tweet") + private val servedTypeStatsReceiver = baseStatsReceiver.scope("ServedType") + private val followedUsersStatsReceiver = baseStatsReceiver.scope("FollowedUsers") private val responseSizeStatsReceiver = baseStatsReceiver.scope("ResponseSize") private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance") @@ -41,50 +51,119 @@ class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver) inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] ): Stitch[Unit] = { val tweetCandidates = CandidatesUtil - .getItemCandidates(inputs.selectedCandidates).filter(_.isCandidateType[TweetCandidate]()) + .getItemCandidates(inputs.selectedCandidates) + .filter(candidate => candidatePipelines.contains(candidate.source)) - val expBucket = inputs.query.params(ExperimentStatsParam) - - recordSuggestTypeStats(tweetCandidates, expBucket) - recordContentBalanceStats(tweetCandidates, expBucket) - recordResponseSizeStats(tweetCandidates, expBucket) + recordAuthorStats( + candidates = tweetCandidates, + authors = inputs.query.params(AuthorListForDataCollectionParam), + tweetLevelAuthors = inputs.query.params(AuthorListForStatsParam) + ) + recordServedTypeStats(tweetCandidates) + recordFollowedUsersStats( + tweetCandidates, + inputs.query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty).size).getOrElse(0), + ) + recordContentBalanceStats(tweetCandidates) + recordResponseSizeStats(tweetCandidates) + logColdContentData(tweetCandidates, inputs.query.getRequiredUserId) Stitch.Unit } - def recordSuggestTypeStats( + def recordAuthorStats( + candidates: Seq[CandidateWithDetails], + authors: Set[Long], + tweetLevelAuthors: Set[Long] + ): Unit = { + val filtered = candidates + .filter { candidate => + candidate.features.getOrElse(AuthorIdFeature, None).exists(authors.contains) && + // Only include original tweets + (!candidate.features.getOrElse(IsRetweetFeature, false)) && + candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty + } + + filtered + .groupBy { candidate => + (getServedType(candidate), candidate.features.get(AuthorIdFeature).get) + } + .foreach { + case ((servedType, authorId), authorCandidates) => + val authorStr = authorId.toString.takeRight(9) + authorStatsReceiver.scope(authorStr).counter(servedType).incr(authorCandidates.size) + } + + filtered.map { candidate => + val authorId = candidate.features.get(AuthorIdFeature).get + val authorStr = authorId.toString.takeRight(9) + authorStatsReceiver.scope(authorStr).counter(candidate.candidateIdLong.toString).incr() + if (tweetLevelAuthors.contains(authorId)) + tweetStatsReceiver.counter(candidate.candidateIdLong.toString.takeRight(10)).incr() + } + } + + def logColdContentData(candidates: Seq[CandidateWithDetails], viewerId: Long): Unit = { + candidates.foreach { candidate => + val servedType = candidate.features.get(ServedTypeFeature) + if (servedType == hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i) { + logger.info("Tier1: " + candidate.candidateIdLong + " viewerId: " + viewerId) + } else if (servedType == hmt.ServedType.ForYouContentExplorationTier2DeepRetrievalI2i) { + logger.info("Tier2: " + candidate.candidateIdLong + " viewerId: " + viewerId) + } + } + } + + def recordServedTypeStats( + candidates: Seq[ItemCandidateWithDetails], + ): Unit = { + candidates.groupBy(getServedType).foreach { + case (servedType, servedTypeCandidates) => + servedTypeStatsReceiver.counter(servedType).incr(servedTypeCandidates.size) + } + } + + def recordFollowedUsersStats( candidates: Seq[ItemCandidateWithDetails], - expBucket: String + followedUsers: Int, ): Unit = { - candidates.groupBy(getSuggestType).foreach { - case (suggestType, suggestTypeCandidates) => - suggestTypeStatsReceiver - .scope(expBucket).counter(suggestType).incr(suggestTypeCandidates.size) + val followedUsersScope = + if (followedUsers < 10) "0-9" + else if (followedUsers < 100) "10-99" + else if (followedUsers < 1000) "100-999" + else if (followedUsers < 10000) "1000-9999" + else ">10000" + + candidates.groupBy(getServedType).foreach { + case (servedType, servedTypeCandidates) => + followedUsersStatsReceiver + .scope(followedUsersScope).counter(servedType) + .incr(servedTypeCandidates.size) } } def recordContentBalanceStats( candidates: Seq[ItemCandidateWithDetails], - expBucket: String ): Unit = { val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true)) - inNetworkStatsReceiver.counter(expBucket).incr(in.size) - outOfNetworkStatsReceiver.counter(expBucket).incr(oon.size) + inNetworkStatsReceiver.counter().incr(in.size) + outOfNetworkStatsReceiver.counter().incr(oon.size) val (reply, original) = candidates.partition(_.features.getOrElse(InReplyToTweetIdFeature, None).isDefined) - replyStatsReceiver.counter(expBucket).incr(reply.size) - originalStatsReceiver.counter(expBucket).incr(original.size) + replyStatsReceiver.counter().incr(reply.size) + originalStatsReceiver.counter().incr(original.size) } def recordResponseSizeStats( candidates: Seq[ItemCandidateWithDetails], - expBucket: String ): Unit = { - if (candidates.size == 0) emptyStatsReceiver.counter(expBucket).incr() - if (candidates.size < 5) lessThan5StatsReceiver.counter(expBucket).incr() - if (candidates.size < 10) lessThan10StatsReceiver.counter(expBucket).incr() + if (candidates.size == 0) emptyStatsReceiver.counter().incr() + if (candidates.size < 5) lessThan5StatsReceiver.counter().incr() + if (candidates.size < 10) lessThan10StatsReceiver.counter().incr() } - private def getSuggestType(candidate: CandidateWithDetails): String = - candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name).getOrElse("None") + private def getServedType(candidate: CandidateWithDetails): String = + candidate.features.get(ServedTypeFeature).name + + override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/VideoServedStatsSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/VideoServedStatsSideEffect.scala new file mode 100644 index 000000000..bdbeb27a1 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/VideoServedStatsSideEffect.scala @@ -0,0 +1,124 @@ +package com.twitter.home_mixer.product.for_you.side_effect + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature +import com.twitter.home_mixer.model.HomeFeatures.ViralContentCreatorFeature +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +case class VideoServedStatsSideEffect( + candidatePipelines: Set[CandidatePipelineIdentifier], + statsReceiver: StatsReceiver) + extends PipelineResultSideEffect[PipelineQuery, Timeline] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("VideoServedStats") + private val baseStatsReceiver = statsReceiver.scope(identifier.toString) + private val videoStatsReceiver = baseStatsReceiver.scope("Video") + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] + ): Stitch[Unit] = { + val tweetCandidates = CandidatesUtil + .getItemCandidates(inputs.selectedCandidates) + .filter(candidate => candidatePipelines.contains(candidate.source)) + + val clientId = inputs.query.clientContext.appId.getOrElse(0L).toString + + // Filter candidates to include only those with video and valid duration + val videoCandidates = tweetCandidates.filter { candidate => + candidate.features.getOrElse(HasVideoFeature, false) && + candidate.features.get(VideoDurationMsFeature).exists(_ > 0) + } + statsReceiver + .scope(clientId).counter("HasVideo") + .incr(videoCandidates.size) + + recordVideoDurationStats(videoCandidates, videoStatsReceiver, clientId) + recordViralContentStats(videoCandidates, videoStatsReceiver, clientId) + Stitch.Unit + } + + def recordViralContentStats( + candidates: Seq[ItemCandidateWithDetails], + statsReceiver: StatsReceiver, + clientId: String + ): Unit = { + + val viralContentCount = candidates.count { candidate => + candidate.features.getOrElse(ViralContentCreatorFeature, false) + } + val viralContentInNetworkCount = candidates.count { candidate => + candidate.features.getOrElse(ViralContentCreatorFeature, false) && + candidate.features.getOrElse(InNetworkFeature, true) + } + val viralContentOutOfNetworkCount = candidates.count { candidate => + candidate.features.getOrElse(ViralContentCreatorFeature, false) && + !candidate.features.getOrElse(InNetworkFeature, true) + } + + statsReceiver + .scope(clientId).counter("ViralContent") + .incr(viralContentCount) + statsReceiver + .scope(clientId) + .counter("ViralContentInNetwork") + .incr(viralContentInNetworkCount) + statsReceiver + .scope(clientId) + .counter("ViralContentOutOfNetwork") + .incr(viralContentOutOfNetworkCount) + + } + + def recordVideoDurationStats( + candidates: Seq[ItemCandidateWithDetails], + statsReceiver: StatsReceiver, + clientId: String + ): Unit = { + + val lte10Sec = candidates.count(_.features.get(VideoDurationMsFeature).exists(_ <= 10000)) + val gt60Sec = candidates.count(_.features.get(VideoDurationMsFeature).exists(_ > 60000)) + val bt10And60Sec = candidates.count { candidate => + candidate.features.get(VideoDurationMsFeature).exists { duration => + duration > 10000 && duration <= 60000 + } + } + val bt60And120Sec = candidates.count { candidate => + candidate.features.get(VideoDurationMsFeature).exists { duration => + duration > 60000 && duration <= 120000 + } + } + val bt120And180Sec = candidates.count { candidate => + candidate.features.get(VideoDurationMsFeature).exists { duration => + duration > 120000 && duration <= 180000 + } + } + val gt180Sec = candidates.count(_.features.get(VideoDurationMsFeature).exists(_ > 180000)) + + statsReceiver + .scope(clientId).counter("VideoLte10Sec").incr(lte10Sec) + statsReceiver + .scope(clientId).counter("VideoGt60Sec").incr(gt60Sec) + statsReceiver + .scope(clientId).counter("VideoBt10And60Sec") + .incr(bt10And60Sec) + statsReceiver + .scope(clientId).counter("VideoBt60And120Sec") + .incr(bt60And120Sec) + statsReceiver + .scope(clientId).counter("VideoBt120And180Sec") + .incr(bt120And180Sec) + statsReceiver + .scope(clientId).counter("VideoGt180Sec").incr(gt180Sec) + } + override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel index 4bbbec94d..7801b587e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel @@ -4,37 +4,48 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", + "home-mixer-features/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/offline_aggregates", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/control_ai", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/user_fav_avg_embeddings", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "src/scala/com/twitter/timelines/prediction/adapters/large_embeddings", + "src/thrift/com/twitter/search:earlybird-scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala index 27278db0c..9be102c14 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala @@ -1,12 +1,16 @@ package com.twitter.home_mixer.product.scored_tweets +import com.twitter.home_mixer.model.HomeFeatures.ServedAuthorIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.SignupCountryFeature +import com.twitter.home_mixer.model.HomeFeatures.SignupSourceFeature import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature +import com.twitter.home_mixer.model.HomeFeatures.UserFollowersCountFeature +import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsForYouRecommendationsFeature import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProductContext import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ServerMaxResultsParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParamConfig import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy import com.twitter.home_mixer.{thriftscala => t} @@ -49,6 +53,11 @@ class ScoredTweetsProductPipelineConfig @Inject() ( val featureMap = FeatureMapBuilder() .add(ServedTweetIdsFeature, context.servedTweetIds.getOrElse(Seq.empty)) .add(TimelineServiceTweetsFeature, context.backfillTweetIds.getOrElse(Seq.empty)) + .add(SignupCountryFeature, context.signupCountryCode) + .add(ViewerAllowsForYouRecommendationsFeature, context.allowForYouRecommendations) + .add(SignupSourceFeature, context.signupSource) + .add(ServedAuthorIdsFeature, context.servedAuthorIds.getOrElse(Map.empty[Long, Seq[Long]])) + .add(UserFollowersCountFeature, context.followerCount) .build() ScoredTweetsQuery( @@ -56,12 +65,13 @@ class ScoredTweetsProductPipelineConfig @Inject() ( clientContext = request.clientContext, pipelineCursor = request.serializedRequestCursor.flatMap(UrtCursorSerializer.deserializeOrderedCursor), - requestedMaxResults = Some(params(ServerMaxResultsParam)), + requestedMaxResults = request.maxResults, debugOptions = request.debugParams.flatMap(_.debugOptions), features = Some(featureMap), deviceContext = context.deviceContext, seenTweetIds = context.seenTweetIds, - qualityFactorStatus = None + qualityFactorStatus = None, + product = product ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala index e3d50040b..c0203dc8e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala @@ -1,81 +1,177 @@ package com.twitter.home_mixer.product.scored_tweets import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.functional_component.feature_hydrator.FollowableUttTopicsQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.CategoryDiversityRescoringFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.DiversityRescoringFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.FeedbackHistoryQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.GizmoduckUserQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.GrokTranslatedPostIsCachedFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.HeartbeatOptimizerParamsHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.HeavyRankerWeightsQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.ImpressedMediaClusterIdsQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.ImpressionBloomFilterQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.ListIdsQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.NaviClientConfigQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.OnPremRealGraphQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.OptimizerWeightsQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.PhoenixRescoringFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphInNetworkScoresQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.RealTimeEntityRealGraphQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.RealTimeInteractionGraphUserVertexQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RequestQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetImpressionsQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.RequestTimeQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.SimClustersUserSparseEmbeddingsQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetTypeMetricsFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinRebuildUserEngagementQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinRebuildUserPositiveQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinUserEngagementQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinUserFollowQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinUserNegativeQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinUserPositiveQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.UnifiedUserActionsUserIdentifierFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.UserActionsQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.UserEngagedGrokCategoriesFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.UserEngagedLanguagesFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.UserHistoryTransformerEmbeddingQueryFeatureHydratorBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator.UserLanguagesFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.UserLargeEmbeddingsFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.UserStateQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.UserUnderstandableLanguagesFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.PartAAggregateQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.PartBAggregateQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates.UserEngagementRealTimeAggregatesFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.user_history.ScoredTweetsUserHistoryEventsQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.filter.ClipImageClusterDeduplicationFilter +import com.twitter.home_mixer.functional_component.filter.ClipVideoClusterDeduplicationFilter import com.twitter.home_mixer.functional_component.filter.FeedbackFatigueFilter +import com.twitter.home_mixer.functional_component.filter.GrokGoreFilter +import com.twitter.home_mixer.functional_component.filter.GrokNsfwFilter +import com.twitter.home_mixer.functional_component.filter.GrokSpamFilter +import com.twitter.home_mixer.functional_component.filter.GrokViolentFilter +import com.twitter.home_mixer.functional_component.filter.HasAuthorFilter +import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter +import com.twitter.home_mixer.functional_component.filter.LocationFilter +import com.twitter.home_mixer.functional_component.filter.MediaIdDeduplicationFilter +import com.twitter.home_mixer.functional_component.filter.MinVideoDurationFilter import com.twitter.home_mixer.functional_component.filter.PreviouslySeenTweetsFilter import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetsFilter +import com.twitter.home_mixer.functional_component.filter.QuoteDeduplicationFilter import com.twitter.home_mixer.functional_component.filter.RejectTweetFromViewerFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter +import com.twitter.home_mixer.functional_component.filter.SlopFilter +import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter +import com.twitter.home_mixer.functional_component.selector.SortFixedPositionContentExplorationMixedCandidates +import com.twitter.home_mixer.functional_component.selector.SortFixedPositionContentExplorationSimclusterColdPostsCandidates +import com.twitter.home_mixer.functional_component.selector.SortFixedPositionDeepRetrievalMixedCandidates import com.twitter.home_mixer.functional_component.side_effect.PublishClientSentImpressionsEventBusSideEffect -import com.twitter.home_mixer.functional_component.side_effect.PublishClientSentImpressionsManhattanSideEffect -import com.twitter.home_mixer.functional_component.side_effect.PublishImpressionBloomFilterSideEffect import com.twitter.home_mixer.functional_component.side_effect.UpdateLastNonPollingTimeSideEffect import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsSupportAccountReplyFeature +import com.twitter.home_mixer.model.HomeFeatures.OonNsfwFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter -import com.twitter.home_mixer.param.HomeMixerFlagName.TargetFetchLatency +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableRealGraphQueryFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildUserEngagementFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildUserPositiveFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinUserNegativeFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinUserPositiveFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableUserFavAvgTextEmbeddingsQueryFeatureParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableUserHistoryTransformerJointBlueEmbeddingFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.CategoryDiversityRescoringParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.TwhinDiversityRescoringParam import com.twitter.home_mixer.param.HomeMixerFlagName.TargetScoringLatency import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsBackfillCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsFrsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsInNetworkCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsContentExplorationCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsDirectUtegCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsListsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsPopularVideosCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsStaticCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsTweetMixerCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsUtegCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird.ScoredTweetsCommunitiesCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird.ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.CachedScoredTweetsQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ListIdsQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealTimeInteractionGraphUserVertexQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RequestTimeQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TwhinUserEngagementQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TwhinUserFollowQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.UserLanguagesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.UserStateQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.PartAAggregateQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.PartBAggregateQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates.UserEngagementRealTimeAggregatesFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.InvalidateCachedScoredTweetsQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.IsColdStartPostFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.LowSignalUserQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.SGSMutuallyFollowedUserHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieVisibilityFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ValidLikedByUserIdsFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.filter.ControlAiExcludeFilter +import com.twitter.home_mixer.product.scored_tweets.filter.ControlAiOnlyIncludeFilter +import com.twitter.home_mixer.product.scored_tweets.filter.CustomSnowflakeIdAgeFilter import com.twitter.home_mixer.product.scored_tweets.filter.DuplicateConversationTweetsFilter -import com.twitter.home_mixer.product.scored_tweets.filter.OutOfNetworkCompetitorFilter -import com.twitter.home_mixer.product.scored_tweets.filter.OutOfNetworkCompetitorURLFilter -import com.twitter.home_mixer.product.scored_tweets.filter.ScoredTweetsSocialContextFilter +import com.twitter.home_mixer.product.scored_tweets.filter.GrokAutoTranslateLanguageFilter +import com.twitter.home_mixer.product.scored_tweets.filter.IsOutOfNetworkColdStartPostFilter +import com.twitter.home_mixer.product.scored_tweets.filter.LanguageFilter +import com.twitter.home_mixer.product.scored_tweets.filter.SGSAuthorFilter import com.twitter.home_mixer.product.scored_tweets.marshaller.ScoredTweetsResponseDomainMarshaller import com.twitter.home_mixer.product.scored_tweets.marshaller.ScoredTweetsResponseTransportMarshaller import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MaxInNetworkResultsParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MaxOutOfNetworkResultsParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.DefaultRequestedMaxResultsParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableClipImageClusterDedupingParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableColdStartFilterParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationSimclusterColdPostsCandidateBoostingParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationCandidatePipelineParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationMixedCandidateBoostingParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableControlAiParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableDeepRetrievalMixedCandidateBoostingParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableHeartbeatOptimizerWeightsParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableMediaClusterDedupingParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableMediaDedupingParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnablePhoenixRescoreParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableRecentEngagementCacheRefreshParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ServerMaxResultsParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMediaClusterDecayParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableRealTimeEntityRealGraphFeaturesParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableTopicSocialProofFeaturesParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableUserActionsFeatureParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableUserEngagedLanguagesFeaturesParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableUserHistoryEventsFeaturesParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableUserIdentifierFeaturesParam import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsHeuristicScoringPipelineConfig +import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsLowSignalScoringPipelineConfig import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsModelScoringPipelineConfig -import com.twitter.home_mixer.product.scored_tweets.selector.KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType +import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsRerankingScoringPipelineConfig +import com.twitter.home_mixer.product.scored_tweets.selector.KeepTopKCandidatesPerCommunity +import com.twitter.home_mixer.product.scored_tweets.side_effect.CacheCandidateFeaturesSideEffect +import com.twitter.home_mixer.product.scored_tweets.side_effect.CacheRequestInfoSideEffect +import com.twitter.home_mixer.product.scored_tweets.side_effect.CacheRetrievalSignalSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.CachedScoredTweetsSideEffect +import com.twitter.home_mixer.product.scored_tweets.side_effect.CommonFeaturesSideEffect +import com.twitter.home_mixer.product.scored_tweets.side_effect.ScoredCandidateFeatureKeysKafkaSideEffect +import com.twitter.home_mixer.product.scored_tweets.side_effect.ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect +import com.twitter.home_mixer.product.scored_tweets.side_effect.ScoredPhoenixCandidatesKafkaSideEffect +import com.twitter.home_mixer.product.scored_tweets.side_effect.ScoredStatsSideEffect +import com.twitter.home_mixer.product.scored_tweets.side_effect.ScoredTweetsDiversityStatsSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.ScribeScoredCandidatesSideEffect -import com.twitter.home_mixer.product.scored_tweets.side_effect.ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect import com.twitter.home_mixer.{thriftscala => t} import com.twitter.inject.annotations.Flag +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.communities.CommunityMembershipsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.ControlAiQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.location.UserLocationQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.user_fav_avg_embeddings.UserFavAvgTextEmbeddingsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.filter.ParamGatedFilter import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.selector.DropDuplicateCandidates -import com.twitter.product_mixer.component_library.selector.DropFilteredMaxCandidates -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates +import com.twitter.product_mixer.component_library.selector.DropRequestedMaxResults import com.twitter.product_mixer.component_library.selector.IdAndClassDuplicationKey import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.PickFirstCandidateMerger +import com.twitter.product_mixer.component_library.selector.SelectConditionally import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates import com.twitter.product_mixer.component_library.selector.sorter.FeatureValueSorter -import com.twitter.product_mixer.core.functional_component.common.AllExceptPipelines import com.twitter.product_mixer.core.functional_component.common.AllPipelines import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator @@ -89,7 +185,6 @@ import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifie import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineConfig @@ -98,55 +193,93 @@ import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig import com.twitter.util.Duration - import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsRecommendationPipelineConfig @Inject() ( - scoredTweetsInNetworkCandidatePipelineConfig: ScoredTweetsInNetworkCandidatePipelineConfig, - scoredTweetsUtegCandidatePipelineConfig: ScoredTweetsUtegCandidatePipelineConfig, + // Candidate pipelines scoredTweetsTweetMixerCandidatePipelineConfig: ScoredTweetsTweetMixerCandidatePipelineConfig, - scoredTweetsFrsCandidatePipelineConfig: ScoredTweetsFrsCandidatePipelineConfig, + scoredTweetsStaticCandidatePipelineConfig: ScoredTweetsStaticCandidatePipelineConfig, scoredTweetsListsCandidatePipelineConfig: ScoredTweetsListsCandidatePipelineConfig, - scoredTweetsPopularVideosCandidatePipelineConfig: ScoredTweetsPopularVideosCandidatePipelineConfig, scoredTweetsBackfillCandidatePipelineConfig: ScoredTweetsBackfillCandidatePipelineConfig, + scoredTweetsEarlybirdInNetworkCandidatePipelineConfig: ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig, + scoredTweetsCommunitiesCandidatePipelineConfig: ScoredTweetsCommunitiesCandidatePipelineConfig, + scoredTweetsDirectUtegCandidatePipelineConfig: ScoredTweetsDirectUtegCandidatePipelineConfig, + scoredTweetsContentExplorationCandidatePipelineConfig: ScoredTweetsContentExplorationCandidatePipelineConfig, cachedScoredTweetsCandidatePipelineConfig: CachedScoredTweetsCandidatePipelineConfig, + // Query feature hydrators + followableUttTopicsQueryFeatureHydrator: FollowableUttTopicsQueryFeatureHydrator, + gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, + controlAiQueryFeatureHydrator: ControlAiQueryFeatureHydrator, + lowSignalUserQueryFeatureHydrator: LowSignalUserQueryFeatureHydrator, + naviClientConfigQueryFeatureHydrator: NaviClientConfigQueryFeatureHydrator, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ScoredTweetsQuery], requestTimeQueryFeatureHydrator: RequestTimeQueryFeatureHydrator, realTimeInteractionGraphUserVertexQueryFeatureHydrator: RealTimeInteractionGraphUserVertexQueryFeatureHydrator, + sgsMutuallyFollowedUserHydrator: SGSMutuallyFollowedUserHydrator, + simClustersUserSparseEmbeddingsQueryFeatureHydrator: SimClustersUserSparseEmbeddingsQueryFeatureHydrator, + userLocationQueryFeatureHydrator: UserLocationQueryFeatureHydrator, userStateQueryFeatureHydrator: UserStateQueryFeatureHydrator, userEngagementRealTimeAggregatesFeatureHydrator: UserEngagementRealTimeAggregatesFeatureHydrator, + twhinRebuildUserEngagementQueryFeatureHydrator: TwhinRebuildUserEngagementQueryFeatureHydrator, + twhinRebuildUserPositiveQueryFeatureHydrator: TwhinRebuildUserPositiveQueryFeatureHydrator, twhinUserEngagementQueryFeatureHydrator: TwhinUserEngagementQueryFeatureHydrator, twhinUserFollowQueryFeatureHydrator: TwhinUserFollowQueryFeatureHydrator, + twhinUserPositiveQueryFeatureHydrator: TwhinUserPositiveQueryFeatureHydrator, + twhinUserNegativeQueryFeatureHydrator: TwhinUserNegativeQueryFeatureHydrator, + userHistoryTransformerEmbeddingQueryFeatureHydratorBuilder: UserHistoryTransformerEmbeddingQueryFeatureHydratorBuilder, cachedScoredTweetsQueryFeatureHydrator: CachedScoredTweetsQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, scoredTweetsModelScoringPipelineConfig: ScoredTweetsModelScoringPipelineConfig, - impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ - ScoredTweetsQuery - ], - manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ - ScoredTweetsQuery - ], + scoredTweetsRerankingScoringPipelineConfig: ScoredTweetsRerankingScoringPipelineConfig, + impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator, memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, + userEngagementsAvgTextEmbeddingsQueryFeatureHydrator: UserFavAvgTextEmbeddingsQueryFeatureHydrator, listIdsQueryFeatureHydrator: ListIdsQueryFeatureHydrator, feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, - publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, + communityMembershipsQueryFeatureHydrator: CommunityMembershipsQueryFeatureHydrator, + tweetypieVisibilityFeatureHydrator: TweetypieVisibilityFeatureHydrator, realGraphInNetworkScoresQueryFeatureHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, realGraphQueryFeatureHydrator: RealGraphQueryFeatureHydrator, + onPremRealGraphQueryFeatureHydrator: OnPremRealGraphQueryFeatureHydrator, + entityRealGraphQueryFeatureHydrator: RealTimeEntityRealGraphQueryFeatureHydrator, + unifiedUserActionsUserIdentifierFeatureHydrator: UnifiedUserActionsUserIdentifierFeatureHydrator, + userEngagedLanguagesFeatureHydrator: UserEngagedLanguagesFeatureHydrator, userLanguagesFeatureHydrator: UserLanguagesFeatureHydrator, + userUnderstandableLanguagesFeatureHydrator: UserUnderstandableLanguagesFeatureHydrator, partAAggregateQueryFeatureHydrator: PartAAggregateQueryFeatureHydrator, partBAggregateQueryFeatureHydrator: PartBAggregateQueryFeatureHydrator, + heavyRankerWeightsQueryFeatureHydrator: HeavyRankerWeightsQueryFeatureHydrator, + userLargeEmbeddingsFeatureHydrator: UserLargeEmbeddingsFeatureHydrator, + userHistoryEventsQueryFeatureHydrator: ScoredTweetsUserHistoryEventsQueryFeatureHydrator, + userActionsQueryFeatureHydrator: UserActionsQueryFeatureHydrator, + impressedMediaClusterIdsQueryFeatureHydrator: ImpressedMediaClusterIdsQueryFeatureHydrator, + heartbeatOptimizerParamsHydrator: HeartbeatOptimizerParamsHydrator, + optimizerWeightsQueryFeatureHydrator: OptimizerWeightsQueryFeatureHydrator, + userEngagedGrokCategoriesFeatureHydrator: UserEngagedGrokCategoriesFeatureHydrator, + grokTranslatedPostIsCachedFeatureHydrator: GrokTranslatedPostIsCachedFeatureHydrator, + isColdStartPostFeatureHydrator: IsColdStartPostFeatureHydrator, + // Filters + invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, + sgsAuthorFilter: SGSAuthorFilter, + // Side effects + cacheCandidateFeaturesSideEffect: CacheCandidateFeaturesSideEffect, cachedScoredTweetsSideEffect: CachedScoredTweetsSideEffect, + cacheRetrievalSignalSideEffect: CacheRetrievalSignalSideEffect, + cacheRequestInfoSideEffect: CacheRequestInfoSideEffect, + scoredCandidateFeatureKeysKafkaSideEffect: ScoredCandidateFeatureKeysKafkaSideEffect, + scoredContentExplorationCandidateScoreFeatureKafkaSideEffect: ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect, + publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, + scoredStatsSideEffect: ScoredStatsSideEffect, + scoredTweetsDiversityStatsSideEffect: ScoredTweetsDiversityStatsSideEffect, + scoredPhoenixCandidatesKafkaSideEffect: ScoredPhoenixCandidatesKafkaSideEffect, + scribeCommonFeaturesSideEffect: CommonFeaturesSideEffect, scribeScoredCandidatesSideEffect: ScribeScoredCandidatesSideEffect, - scribeServedCommonFeaturesAndCandidateFeaturesSideEffect: ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect, updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ ScoredTweetsQuery, ScoredTweetsResponse ], - @Flag(TargetFetchLatency) targetFetchLatency: Duration, @Flag(TargetScoringLatency) targetScoringLatency: Duration) extends RecommendationPipelineConfig[ ScoredTweetsQuery, @@ -159,46 +292,171 @@ class ScoredTweetsRecommendationPipelineConfig @Inject() ( RecommendationPipelineIdentifier("ScoredTweets") private val SubscriptionReplyFilterId = "SubscriptionReply" - private val MaxBackfillTweets = 50 + private val OutOfNetworkNSFWFilterId = "OutOfNetworkNSFW" private val scoringStep = RecommendationPipelineConfig.scoringPipelinesStep override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ScoredTweetsQuery]] = Seq( + naviClientConfigQueryFeatureHydrator, requestQueryFeatureHydrator, realGraphInNetworkScoresQueryFeatureHydrator, cachedScoredTweetsQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator, - ParamGatedQueryFeatureHydrator( - EnableImpressionBloomFilter, - impressionBloomFilterQueryFeatureHydrator - ), - manhattanTweetImpressionsQueryFeatureHydrator, memcacheTweetImpressionsQueryFeatureHydrator, listIdsQueryFeatureHydrator, + communityMembershipsQueryFeatureHydrator, userStateQueryFeatureHydrator, - AsyncQueryFeatureHydrator(scoringStep, feedbackHistoryQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, realGraphQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, requestTimeQueryFeatureHydrator), + lowSignalUserQueryFeatureHydrator, + feedbackHistoryQueryFeatureHydrator, + requestTimeQueryFeatureHydrator, + userUnderstandableLanguagesFeatureHydrator, + ParamGatedQueryFeatureHydrator( + EnableContentExplorationCandidatePipelineParam, + userEngagedGrokCategoriesFeatureHydrator + ), + AsyncQueryFeatureHydrator( + RecommendationPipelineConfig.globalFiltersStep, + userLocationQueryFeatureHydrator + ), + AsyncQueryFeatureHydrator( + RecommendationPipelineConfig.globalFiltersStep, + impressionBloomFilterQueryFeatureHydrator + ), + AsyncParamGatedQueryFeatureHydrator( + EnableRealGraphQueryFeaturesParam, + scoringStep, + realGraphQueryFeatureHydrator + ), + AsyncParamGatedQueryFeatureHydrator( + EnableRealGraphQueryFeaturesParam, + scoringStep, + onPremRealGraphQueryFeatureHydrator + ), + AsyncParamGatedQueryFeatureHydrator( + EnableRealTimeEntityRealGraphFeaturesParam, + scoringStep, + entityRealGraphQueryFeatureHydrator + ), + AsyncParamGatedQueryFeatureHydrator( + EnableTwhinRebuildUserEngagementFeaturesParam, + scoringStep, + twhinRebuildUserEngagementQueryFeatureHydrator + ), + AsyncParamGatedQueryFeatureHydrator( + EnableTwhinRebuildUserPositiveFeaturesParam, + scoringStep, + twhinRebuildUserPositiveQueryFeatureHydrator + ), + AsyncParamGatedQueryFeatureHydrator( + EnableTwhinUserNegativeFeaturesParam, + scoringStep, + twhinUserNegativeQueryFeatureHydrator + ), + AsyncParamGatedQueryFeatureHydrator( + EnableTwhinUserPositiveFeaturesParam, + scoringStep, + twhinUserPositiveQueryFeatureHydrator + ), + AsyncQueryFeatureHydrator( + scoringStep, + userHistoryTransformerEmbeddingQueryFeatureHydratorBuilder.buildHomeBlueHydrator() + ), + AsyncQueryFeatureHydrator( + scoringStep, + userHistoryTransformerEmbeddingQueryFeatureHydratorBuilder.buildHomeGreenHydrator() + ), + AsyncParamGatedQueryFeatureHydrator( + EnableUserHistoryTransformerJointBlueEmbeddingFeaturesParam, + scoringStep, + userHistoryTransformerEmbeddingQueryFeatureHydratorBuilder.buildJointBlueHydrator() + ), + AsyncParamGatedQueryFeatureHydrator( + EnableTopicSocialProofFeaturesParam, + scoringStep, + followableUttTopicsQueryFeatureHydrator + ), + AsyncParamGatedQueryFeatureHydrator( + EnableUserEngagedLanguagesFeaturesParam, + scoringStep, + userEngagedLanguagesFeatureHydrator + ), + AsyncParamGatedQueryFeatureHydrator( + EnableLargeEmbeddingsFeatureHydrationParam, + scoringStep, + userLargeEmbeddingsFeatureHydrator, + ), + AsyncParamGatedQueryFeatureHydrator( + EnableUserHistoryEventsFeaturesParam, + scoringStep, + userHistoryEventsQueryFeatureHydrator, + ), + AsyncParamGatedQueryFeatureHydrator( + EnableUserActionsFeatureParam, + scoringStep, + userActionsQueryFeatureHydrator + ), + AsyncParamGatedQueryFeatureHydrator( + EnableMediaClusterDecayParam, + scoringStep, + impressedMediaClusterIdsQueryFeatureHydrator, + ), + AsyncParamGatedQueryFeatureHydrator( + EnableUserIdentifierFeaturesParam, + scoringStep, + unifiedUserActionsUserIdentifierFeatureHydrator + ), AsyncQueryFeatureHydrator(scoringStep, userLanguagesFeatureHydrator), + AsyncQueryFeatureHydrator(scoringStep, sgsMutuallyFollowedUserHydrator), AsyncQueryFeatureHydrator(scoringStep, userEngagementRealTimeAggregatesFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, realTimeInteractionGraphUserVertexQueryFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, twhinUserFollowQueryFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, twhinUserEngagementQueryFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, partAAggregateQueryFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, partBAggregateQueryFeatureHydrator), + AsyncQueryFeatureHydrator(scoringStep, heavyRankerWeightsQueryFeatureHydrator), + ParamGatedQueryFeatureHydrator( + EnableHeartbeatOptimizerWeightsParam, + heartbeatOptimizerParamsHydrator + ), + AsyncQueryFeatureHydrator(scoringStep, simClustersUserSparseEmbeddingsQueryFeatureHydrator), + AsyncParamGatedQueryFeatureHydrator( + EnableControlAiParam, + scoringStep, + controlAiQueryFeatureHydrator + ), + AsyncParamGatedQueryFeatureHydrator( + EnableUserFavAvgTextEmbeddingsQueryFeatureParam, + RecommendationPipelineConfig.resultSideEffectsStep, + userEngagementsAvgTextEmbeddingsQueryFeatureHydrator + ), + ) + + override val fetchQueryFeaturesPhase2: Seq[QueryFeatureHydrator[ScoredTweetsQuery]] = Seq( + AsyncParamGatedQueryFeatureHydrator( + EnableHeartbeatOptimizerWeightsParam, + scoringStep, + optimizerWeightsQueryFeatureHydrator + ), + ParamGatedQueryFeatureHydrator( + EnableRecentEngagementCacheRefreshParam, + InvalidateCachedScoredTweetsQueryFeatureHydrator + ) ) override val candidatePipelines: Seq[ CandidatePipelineConfig[ScoredTweetsQuery, _, _, TweetCandidate] ] = Seq( + // Order matters. Duplicated candidates take the first occurrence in the pipelines when merger + // strategy is PickFirstCandidateMerger. + scoredTweetsStaticCandidatePipelineConfig, cachedScoredTweetsCandidatePipelineConfig, - scoredTweetsInNetworkCandidatePipelineConfig, - scoredTweetsUtegCandidatePipelineConfig, + scoredTweetsEarlybirdInNetworkCandidatePipelineConfig, + scoredTweetsDirectUtegCandidatePipelineConfig, scoredTweetsTweetMixerCandidatePipelineConfig, - scoredTweetsFrsCandidatePipelineConfig, + scoredTweetsContentExplorationCandidatePipelineConfig, scoredTweetsListsCandidatePipelineConfig, - scoredTweetsPopularVideosCandidatePipelineConfig, - scoredTweetsBackfillCandidatePipelineConfig + scoredTweetsBackfillCandidatePipelineConfig, + scoredTweetsCommunitiesCandidatePipelineConfig ) override val postCandidatePipelinesSelectors: Seq[Selector[ScoredTweetsQuery]] = Seq( @@ -212,8 +470,11 @@ class ScoredTweetsRecommendationPipelineConfig @Inject() ( override val globalFilters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( // sort these to have the "cheaper" filters run first + CustomSnowflakeIdAgeFilter(StaticParam(48.hours)), + HasAuthorFilter, RejectTweetFromViewerFilter, RetweetDeduplicationFilter, + LocationFilter, PreviouslySeenTweetsFilter, PreviouslyServedTweetsFilter, PredicateFeatureFilter.fromPredicate( @@ -223,47 +484,38 @@ class ScoredTweetsRecommendationPipelineConfig @Inject() ( features.getOrElse(ExclusiveConversationAuthorIdFeature, None).isEmpty } ), - FeedbackFatigueFilter + FeedbackFatigueFilter, + invalidSubscriptionTweetFilter, + sgsAuthorFilter ) override val candidatePipelineFailOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( + scoredTweetsStaticCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, cachedScoredTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsInNetworkCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsUtegCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, scoredTweetsTweetMixerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsFrsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, scoredTweetsListsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsPopularVideosCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsBackfillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always + scoredTweetsBackfillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + scoredTweetsEarlybirdInNetworkCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + scoredTweetsCommunitiesCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + scoredTweetsDirectUtegCandidatePipelineConfig.identifier -> FailOpenPolicy.Always ) override val scoringPipelineFailOpenPolicies: Map[ScoringPipelineIdentifier, FailOpenPolicy] = Map( - ScoredTweetsHeuristicScoringPipelineConfig.identifier -> FailOpenPolicy.Always + ScoredTweetsHeuristicScoringPipelineConfig.identifier -> FailOpenPolicy.Always, + ScoredTweetsLowSignalScoringPipelineConfig.identifier -> FailOpenPolicy.Always ) - private val candidatePipelineQualityFactorConfig = LinearLatencyQualityFactorConfig( + private val scoringPipelineQualityFactorConfig = LinearLatencyQualityFactorConfig( qualityFactorBounds = BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 0.95), initialDelay = 60.seconds, - targetLatency = targetFetchLatency, + targetLatency = targetScoringLatency, targetLatencyPercentile = 95.0, delta = 0.00125 ) - private val scoringPipelineQualityFactorConfig = - candidatePipelineQualityFactorConfig.copy(targetLatency = targetScoringLatency) - override val qualityFactorConfigs: Map[ComponentIdentifier, QualityFactorConfig] = Map( - // candidate pipelines - scoredTweetsInNetworkCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsUtegCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsTweetMixerCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsFrsCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsListsCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsPopularVideosCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsBackfillCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - // scoring pipelines scoredTweetsModelScoringPipelineConfig.identifier -> scoringPipelineQualityFactorConfig, ) @@ -271,61 +523,124 @@ class ScoredTweetsRecommendationPipelineConfig @Inject() ( Seq( // scoring pipeline - run on non-cached candidates only since cached ones are already scored scoredTweetsModelScoringPipelineConfig, + scoredTweetsRerankingScoringPipelineConfig, // re-scoring pipeline - run on all candidates since these are request specific - ScoredTweetsHeuristicScoringPipelineConfig + ScoredTweetsHeuristicScoringPipelineConfig, + ScoredTweetsLowSignalScoringPipelineConfig, ) override val postScoringFilters = Seq( - ScoredTweetsSocialContextFilter, - OutOfNetworkCompetitorFilter, - OutOfNetworkCompetitorURLFilter, + MinVideoDurationFilter, + ParamGatedFilter( + EnableControlAiParam, + ControlAiExcludeFilter + ), + ParamGatedFilter( + EnableControlAiParam, + ControlAiOnlyIncludeFilter + ), + SlopFilter, + GrokGoreFilter, + GrokNsfwFilter, + GrokSpamFilter, + GrokViolentFilter, + LanguageFilter, + GrokAutoTranslateLanguageFilter, DuplicateConversationTweetsFilter, PredicateFeatureFilter.fromPredicate( FilterIdentifier("IsSupportAccountReply"), shouldKeepCandidate = { features => !features.getOrElse(IsSupportAccountReplyFeature, false) - }) + } + ), ) override val resultSelectors: Seq[Selector[ScoredTweetsQuery]] = Seq( - KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType(AllPipelines), + KeepTopKCandidatesPerCommunity(AllPipelines), UpdateSortCandidates(AllPipelines, FeatureValueSorter.descending(ScoreFeature)), - DropFilteredMaxCandidates( - pipelineScope = - AllExceptPipelines(Set(scoredTweetsBackfillCandidatePipelineConfig.identifier)), - filter = { - case ItemCandidateWithDetails(_, _, features) => - features.getOrElse(InNetworkFeature, false) - case _ => false - }, - maxSelectionsParam = MaxInNetworkResultsParam - ), - DropFilteredMaxCandidates( - pipelineScope = AllPipelines, - filter = { - case ItemCandidateWithDetails(_, _, features) => - !features.getOrElse(InNetworkFeature, false) - case _ => false - }, - maxSelectionsParam = MaxOutOfNetworkResultsParam + SelectConditionally.paramGated( + SortFixedPositionContentExplorationMixedCandidates, + EnableContentExplorationMixedCandidateBoostingParam ), - DropMaxCandidates( - candidatePipeline = scoredTweetsBackfillCandidatePipelineConfig.identifier, - maxSelectionsParam = StaticParam(MaxBackfillTweets) + SelectConditionally.paramGated( + SortFixedPositionDeepRetrievalMixedCandidates, + EnableDeepRetrievalMixedCandidateBoostingParam ), - InsertAppendResults(AllPipelines) + SelectConditionally.paramGated( + SortFixedPositionContentExplorationSimclusterColdPostsCandidates, + EnableContentExplorationSimclusterColdPostsCandidateBoostingParam + ), + InsertAppendResults(AllPipelines), + DropRequestedMaxResults( + defaultRequestedMaxResultsParam = DefaultRequestedMaxResultsParam, + serverMaxResultsParam = ServerMaxResultsParam + ) + ) + + override val postSelectionFeatureHydration = Seq( + grokTranslatedPostIsCachedFeatureHydrator, + tweetypieVisibilityFeatureHydrator, + TweetTypeMetricsFeatureHydrator, + ParamGatedBulkCandidateFeatureHydrator( + EnablePhoenixRescoreParam, + PhoenixRescoringFeatureHydrator + ), + ParamGatedBulkCandidateFeatureHydrator( + TwhinDiversityRescoringParam, + DiversityRescoringFeatureHydrator + ), + ParamGatedBulkCandidateFeatureHydrator( + CategoryDiversityRescoringParam, + CategoryDiversityRescoringFeatureHydrator + ), + ParamGatedCandidateFeatureHydrator( + EnableColdStartFilterParam, + isColdStartPostFeatureHydrator + ), + ValidLikedByUserIdsFeatureHydrator + ) + + override val postSelectionFilters = Seq( + TweetHydrationFilter, + PredicateFeatureFilter.fromPredicate( + FilterIdentifier(OutOfNetworkNSFWFilterId), + shouldKeepCandidate = { features => !features.getOrElse(OonNsfwFeature, false) } + ), + QuoteDeduplicationFilter, + ParamGatedFilter( + EnableMediaDedupingParam, + MediaIdDeduplicationFilter + ), + ParamGatedFilter( + EnableMediaClusterDedupingParam, + ClipVideoClusterDeduplicationFilter + ), + ParamGatedFilter( + EnableClipImageClusterDedupingParam, + ClipImageClusterDeduplicationFilter + ), + ParamGatedFilter( + EnableColdStartFilterParam, + IsOutOfNetworkColdStartPostFilter + ) ) override val resultSideEffects: Seq[ PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] ] = Seq( + cacheCandidateFeaturesSideEffect, cachedScoredTweetsSideEffect, + scoredCandidateFeatureKeysKafkaSideEffect, + scoredContentExplorationCandidateScoreFeatureKafkaSideEffect, + scoredPhoenixCandidatesKafkaSideEffect, publishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect, + scoredStatsSideEffect, + scoredTweetsDiversityStatsSideEffect, + scribeCommonFeaturesSideEffect, scribeScoredCandidatesSideEffect, - scribeServedCommonFeaturesAndCandidateFeaturesSideEffect, - updateLastNonPollingTimeSideEffect + updateLastNonPollingTimeSideEffect, + cacheRetrievalSignalSideEffect, + cacheRequestInfoSideEffect ) override val domainMarshaller: DomainMarshaller[ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel index c5bb294a1..31dd89c51 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel @@ -4,12 +4,12 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", @@ -20,11 +20,12 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweet_mixer", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/uteg", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module:test-user-mapper", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "src/thrift/com/twitter/timelineranker:thrift-scala", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala index 65a42cbee..92eb69edd 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala @@ -1,12 +1,16 @@ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline +import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig._ import com.twitter.home_mixer.product.scored_tweets.candidate_source.CachedScoredTweetsCandidateSource +import com.twitter.home_mixer.product.scored_tweets.gate.RecentFeedbackCheckGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.response_transformer.CachedScoredTweetsResponseFeatureTransformer import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer @@ -30,6 +34,11 @@ class CachedScoredTweetsCandidatePipelineConfig @Inject() ( override val identifier: CandidatePipelineIdentifier = Identifier + override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( + NonEmptySeqFeatureGate(CachedScoredTweetsFeature), + RecentFeedbackCheckGate + ) + override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, ScoredTweetsQuery diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala index c34031c78..9af07a5df 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala @@ -1,17 +1,16 @@ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator +import com.twitter.home_mixer.functional_component.filter.HasAuthorFilter import com.twitter.home_mixer.functional_component.filter.ReplyFilter -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate +import com.twitter.home_mixer.product.scored_tweets.gate.DenyLowSignalUserGate import com.twitter.home_mixer.product.scored_tweets.gate.MinTimeSinceLastRequestGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableBackfillCandidatePipelineParam import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsBackfillResponseFeatureTransformer -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter +import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource @@ -24,16 +23,14 @@ import com.twitter.product_mixer.core.functional_component.transformer.Candidate import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.decider.DeciderParam import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsBackfillCandidatePipelineConfig @Inject() ( - tweetypieStaticEntitiesHydrator: TweetypieStaticEntitiesFeatureHydrator) + tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, ScoredTweetsQuery, @@ -44,19 +41,15 @@ class ScoredTweetsBackfillCandidatePipelineConfig @Inject() ( override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsBackfill") - private val HasAuthorFilterId = "HasAuthor" - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableBackfillParam) - override val supportedClientParam: Option[FSParam[Boolean]] = Some(EnableBackfillCandidatePipelineParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinTimeSinceLastRequestGate, + DenyLowSignalUserGate, NonEmptySeqFeatureGate(TimelineServiceTweetsFeature), - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) + EmptySeqFeatureGate(CachedScoredTweetsFeature) ) override val queryTransformer: CandidatePipelineQueryTransformer[ @@ -82,13 +75,10 @@ class ScoredTweetsBackfillCandidatePipelineConfig @Inject() ( override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq(tweetypieStaticEntitiesHydrator) + ] = Seq(tweetEntityServiceFeatureHydrator) override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( ReplyFilter, - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(HasAuthorFilterId), - shouldKeepCandidate = _.getOrElse(AuthorIdFeature, None).isDefined - ) + HasAuthorFilter ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsContentExplorationCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsContentExplorationCandidatePipelineConfig.scala new file mode 100644 index 000000000..016e6262e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsContentExplorationCandidatePipelineConfig.scala @@ -0,0 +1,75 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline + +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator +import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature +import com.twitter.home_mixer.product.scored_tweets.candidate_source.ContentExplorationCandidateResponse +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.candidate_source.ContentExplorationCandidateSource +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationCandidatePipelineParam +import com.twitter.home_mixer.product.scored_tweets.query_transformer.ContentExplorationQueryRequest +import com.twitter.home_mixer.product.scored_tweets.query_transformer.ContentExplorationQueryTransformer +import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsContentExplorationResponseFeatureTransformer +import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Candidate Pipeline Config that fetches tweets from the content exploration post source + */ +@Singleton +class ScoredTweetsContentExplorationCandidatePipelineConfig @Inject() ( + contentExplorationCandidateSource: ContentExplorationCandidateSource, + tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator) + extends CandidatePipelineConfig[ + ScoredTweetsQuery, + ContentExplorationQueryRequest, + ContentExplorationCandidateResponse, + TweetCandidate + ] { + + override val supportedClientParam: Option[FSParam[Boolean]] = Some( + EnableContentExplorationCandidatePipelineParam) + + override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( + EmptySeqFeatureGate(CachedScoredTweetsFeature)) + + override val identifier: CandidatePipelineIdentifier = + ScoredTweetsContentExplorationCandidatePipelineConfig.identifier + + override val candidateSource: ContentExplorationCandidateSource = + contentExplorationCandidateSource + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ScoredTweetsQuery, + ContentExplorationQueryRequest + ] = ContentExplorationQueryTransformer + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[ContentExplorationCandidateResponse] + ] = Seq(ScoredTweetsContentExplorationResponseFeatureTransformer) + + override val postFilterFeatureHydration: Seq[ + BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] + ] = Seq(tweetEntityServiceFeatureHydrator) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + ContentExplorationCandidateResponse, + TweetCandidate + ] = { sourceResult => + TweetCandidate(id = sourceResult.tweetId) + } +} + +object ScoredTweetsContentExplorationCandidatePipelineConfig { + val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ScoredTweetsContentExploration") +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsDirectUtegCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsDirectUtegCandidatePipelineConfig.scala new file mode 100644 index 000000000..7b5838f4e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsDirectUtegCandidatePipelineConfig.scala @@ -0,0 +1,89 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline + +import com.twitter.home_mixer.functional_component.feature_hydrator.EarlybirdSearchResultFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator +import com.twitter.home_mixer.functional_component.gate.AllowForYouRecommendationsGate +import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature +import com.twitter.home_mixer.product.scored_tweets.filter.FollowedAuthorFilter +import com.twitter.home_mixer.product.scored_tweets.filter.OONReplyFilter +import com.twitter.home_mixer.product.scored_tweets.filter.UtegMinFavCountFilter +import com.twitter.home_mixer.product.scored_tweets.filter.UtegTopKFilter +import com.twitter.home_mixer.product.scored_tweets.gate.DenyLowSignalUserGate +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidateSourceParams +import com.twitter.home_mixer.product.scored_tweets.query_transformer.UtegQueryTransformer +import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsDirectUtegResponseFeatureTransformer +import com.twitter.product_mixer.component_library.candidate_source.uteg.UtegCandidateSource +import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} +import com.twitter.timelines.configapi.FSParam +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ScoredTweetsDirectUtegCandidatePipelineConfig @Inject() ( + utegCandidateSource: UtegCandidateSource, + tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator, + earlybirdSearchResultFeatureHydrator: EarlybirdSearchResultFeatureHydrator) + extends CandidatePipelineConfig[ + ScoredTweetsQuery, + uteg.RecommendTweetEntityRequest, + uteg.TweetRecommendation, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ScoredTweetsDirectUteg") + + override val supportedClientParam: Option[FSParam[Boolean]] = Some( + CandidateSourceParams.EnableUTEGCandidateSourceParam) + + override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( + AllowForYouRecommendationsGate, + DenyLowSignalUserGate, + EmptySeqFeatureGate(CachedScoredTweetsFeature) + ) + + override val candidateSource: BaseCandidateSource[ + uteg.RecommendTweetEntityRequest, + uteg.TweetRecommendation + ] = utegCandidateSource + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ScoredTweetsQuery, + uteg.RecommendTweetEntityRequest + ] = UtegQueryTransformer(identifier) + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] + ] = Seq( + tweetEntityServiceFeatureHydrator, + earlybirdSearchResultFeatureHydrator + ) + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[uteg.TweetRecommendation] + ] = Seq(ScoredTweetsDirectUtegResponseFeatureTransformer) + + override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( + FollowedAuthorFilter, + UtegMinFavCountFilter, + OONReplyFilter, + UtegTopKFilter, + ) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + uteg.TweetRecommendation, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala index 1161a8278..97f207c25 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala @@ -1,15 +1,16 @@ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.ListIdsFeature +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator import com.twitter.home_mixer.functional_component.filter.ReplyFilter import com.twitter.home_mixer.functional_component.filter.RetweetFilter +import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature +import com.twitter.home_mixer.product.scored_tweets.candidate_source.ListTweet import com.twitter.home_mixer.product.scored_tweets.candidate_source.ListsCandidateSource -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ListIdsFeature -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ListNameFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsListsResponseFeatureTransformer +import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource @@ -21,7 +22,6 @@ import com.twitter.product_mixer.core.functional_component.transformer.Candidate import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.decider.DeciderParam import com.twitter.timelineservice.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @@ -29,11 +29,12 @@ import javax.inject.Singleton @Singleton class ScoredTweetsListsCandidatePipelineConfig @Inject() ( listsCandidateSource: ListsCandidateSource, - tweetypieStaticEntitiesHydrator: TweetypieStaticEntitiesFeatureHydrator) + listNameFeatureHydrator: ListNameFeatureHydrator, + tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, Seq[t.TimelineQuery], - t.Tweet, + ListTweet, TweetCandidate ] { @@ -42,12 +43,9 @@ class ScoredTweetsListsCandidatePipelineConfig @Inject() ( private val MaxTweetsToFetchPerList = 20 - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableListsParam) - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( NonEmptySeqFeatureGate(ListIdsFeature), - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) + EmptySeqFeatureGate(CachedScoredTweetsFeature) ) override val queryTransformer: CandidatePipelineQueryTransformer[ @@ -66,20 +64,23 @@ class ScoredTweetsListsCandidatePipelineConfig @Inject() ( } } - override def candidateSource: CandidateSource[Seq[t.TimelineQuery], t.Tweet] = + override def candidateSource: CandidateSource[Seq[t.TimelineQuery], ListTweet] = listsCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.Tweet] + CandidateFeatureTransformer[ListTweet] ] = Seq(ScoredTweetsListsResponseFeatureTransformer) - override val resultTransformer: CandidatePipelineResultsTransformer[t.Tweet, TweetCandidate] = { - sourceResult => TweetCandidate(id = sourceResult.statusId) + override val resultTransformer: CandidatePipelineResultsTransformer[ListTweet, TweetCandidate] = { + sourceResult => TweetCandidate(id = sourceResult.tweet.statusId) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq(tweetypieStaticEntitiesHydrator) + ] = Seq( + listNameFeatureHydrator, + tweetEntityServiceFeatureHydrator, + ) override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq(ReplyFilter, RetweetFilter) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsStaticCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsStaticCandidatePipelineConfig.scala new file mode 100644 index 000000000..af499ac2b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsStaticCandidatePipelineConfig.scala @@ -0,0 +1,77 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline + +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator +import com.twitter.home_mixer.functional_component.gate.AllowForYouRecommendationsGate +import com.twitter.home_mixer.model.HomeFeatures.SignupCountryFeature +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsStaticCandidatePipelineConfig.Identifier +import com.twitter.home_mixer.product.scored_tweets.candidate_source.StaticPostsCandidateSource +import com.twitter.home_mixer.product.scored_tweets.candidate_source.StaticSourceRequest +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidateSourceParams +import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsStaticResponseFeatureTransformer +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ScoredTweetsStaticCandidatePipelineConfig @Inject() ( + staticPostsCandidateSource: StaticPostsCandidateSource, + tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator) + extends CandidatePipelineConfig[ + ScoredTweetsQuery, + StaticSourceRequest, + Long, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = Identifier + + override val gates: Seq[Gate[ScoredTweetsQuery]] = + Seq(AllowForYouRecommendationsGate) + + override val supportedClientParam: Option[FSParam[Boolean]] = + Some(CandidateSourceParams.EnableStaticSourceParam) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ScoredTweetsQuery, + StaticSourceRequest + ] = { query => + val following = + query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).getOrElse(Seq.empty).toSet + val countryCode = query.clientContext.countryCode + val signupCountryCode = query.features.flatMap(_.getOrElse(SignupCountryFeature, None)) + val countryCodes = Seq(countryCode, signupCountryCode).flatten + StaticSourceRequest(countryCodes = countryCodes, following = following) + } + + override def candidateSource: CandidateSource[StaticSourceRequest, Long] = + staticPostsCandidateSource + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] + ] = Seq(tweetEntityServiceFeatureHydrator) + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[Long] + ] = Seq(ScoredTweetsStaticResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + Long, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult) } +} + +object ScoredTweetsStaticCandidatePipelineConfig { + val SourceIdentifier = "ScoredTweetsStatic" + val Identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier(SourceIdentifier) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala index df16e3f11..f2a9fc0b9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala @@ -1,17 +1,19 @@ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator +import com.twitter.home_mixer.functional_component.gate.AllowForYouRecommendationsGate +import com.twitter.home_mixer.product.scored_tweets.filter.ExtendedDirectedAtFilter +import com.twitter.home_mixer.product.scored_tweets.filter.QualifiedRepliesFilter import com.twitter.tweet_mixer.{thriftscala => t} -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FetchParams import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsTweetMixerResponseFeatureTransformer import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.product_mixer.component_library.candidate_source.tweet_mixer.TweetMixerCandidateSource -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.module.TestUserMapper import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter @@ -21,10 +23,8 @@ import com.twitter.product_mixer.core.functional_component.transformer.Candidate import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.decider.DeciderParam - import javax.inject.Inject import javax.inject.Singleton @@ -34,7 +34,8 @@ import javax.inject.Singleton @Singleton class ScoredTweetsTweetMixerCandidatePipelineConfig @Inject() ( tweetMixerCandidateSource: TweetMixerCandidateSource, - tweetypieStaticEntitiesFeatureHydrator: TweetypieStaticEntitiesFeatureHydrator) + tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator, + testUserMapper: TestUserMapper) extends CandidatePipelineConfig[ ScoredTweetsQuery, t.TweetMixerRequest, @@ -42,59 +43,61 @@ class ScoredTweetsTweetMixerCandidatePipelineConfig @Inject() ( TweetCandidate ] { - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsTweetMixer") - - val HasAuthorFilterId = "HasAuthor" - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableTweetMixerParam) + import ScoredTweetsTweetMixerCandidatePipelineConfig._ + override val identifier: CandidatePipelineIdentifier = Identifier override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam), + AllowForYouRecommendationsGate, + MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) ) override val candidateSource: BaseCandidateSource[t.TweetMixerRequest, t.TweetResult] = tweetMixerCandidateSource - private val MaxTweetsToFetch = 400 + def getClientContext( + query: ScoredTweetsQuery + ): ClientContext = { + if (testUserMapper.isTestUser(query.clientContext)) testUserMapper(query.clientContext) + else query.clientContext + } override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, t.TweetMixerRequest ] = { query => - val maxCount = (query.getQualityFactorCurrentValue(identifier) * MaxTweetsToFetch).toInt - val excludedTweetIds = query.features.map( CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweets(_, identifier)) t.TweetMixerRequest( - clientContext = ClientContextMarshaller(query.clientContext), + clientContext = ClientContextMarshaller(getClientContext(query)), product = t.Product.HomeRecommendedTweets, productContext = Some( t.ProductContext.HomeRecommendedTweetsProductContext( t.HomeRecommendedTweetsProductContext(excludedTweetIds = excludedTweetIds.map(_.toSet)))), - maxResults = Some(maxCount) + maxResults = Some(query.params(FetchParams.TweetMixerMaxTweetsToFetchParam)) ) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq(tweetypieStaticEntitiesFeatureHydrator) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.TweetResult] - ] = Seq(ScoredTweetsTweetMixerResponseFeatureTransformer) + ] = Seq(tweetEntityServiceFeatureHydrator) override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(HasAuthorFilterId), - shouldKeepCandidate = _.getOrElse(AuthorIdFeature, None).isDefined - ) + QualifiedRepliesFilter, + ExtendedDirectedAtFilter ) + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[t.TweetResult] + ] = Seq(ScoredTweetsTweetMixerResponseFeatureTransformer()) + override val resultTransformer: CandidatePipelineResultsTransformer[ t.TweetResult, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } } + +object ScoredTweetsTweetMixerCandidatePipelineConfig { + val Identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ScoredTweetsTweetMixer") +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel index 618c4a081..cfdf69ccd 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel @@ -4,18 +4,24 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "src/thrift/com/twitter/search:earlybird-scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsCommunitiesCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsCommunitiesCandidatePipelineConfig.scala new file mode 100644 index 000000000..cce102e97 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsCommunitiesCandidatePipelineConfig.scala @@ -0,0 +1,76 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird + +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator +import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidateSourceParams +import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.CommunitiesEarlybirdQueryTransformer +import com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird.ScoredTweetsCommunitiesResponseFeatureTransformer +import com.twitter.product_mixer.component_library.candidate_source.earlybird.EarlybirdTweetCandidateSource +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.communities.CommunityNamesFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.communities.CommunityMembershipsFeature +import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate +import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.search.earlybird.{thriftscala => t} +import com.twitter.timelines.configapi.FSParam +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ScoredTweetsCommunitiesCandidatePipelineConfig @Inject() ( + earlybirdTweetCandidateSource: EarlybirdTweetCandidateSource, + communityNamesFeatureHydrator: CommunityNamesFeatureHydrator, + communitiesEarlybirdQueryTransformer: CommunitiesEarlybirdQueryTransformer, + tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator) + extends CandidatePipelineConfig[ + ScoredTweetsQuery, + t.EarlybirdRequest, + t.ThriftSearchResult, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ScoredTweetsCommunities") + + override val supportedClientParam: Option[FSParam[Boolean]] = + Some(CandidateSourceParams.EnableCommunitiesCandidateSourceParam) + + override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( + NonEmptySeqFeatureGate(CommunityMembershipsFeature), + EmptySeqFeatureGate(CachedScoredTweetsFeature) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ScoredTweetsQuery, + t.EarlybirdRequest + ] = communitiesEarlybirdQueryTransformer + + override def candidateSource: BaseCandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] = + earlybirdTweetCandidateSource + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[t.ThriftSearchResult] + ] = Seq(ScoredTweetsCommunitiesResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + t.ThriftSearchResult, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult.id) } + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] + ] = Seq(tweetEntityServiceFeatureHydrator) + + override val postFilterFeatureHydration: Seq[ + BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] + ] = Seq(communityNamesFeatureHydrator) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala index acb26ce42..a3977615f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala @@ -1,14 +1,25 @@ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird import com.twitter.finagle.thrift.ClientId -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.candidate_source.EarlybirdRealtimeCGTweetCandidateSource +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FollowedUserScoresFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.IsExtendedReplyFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.filter.ExtendedDirectedAtFilter +import com.twitter.home_mixer.product.scored_tweets.filter.QualifiedRepliesFilter import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidateSourceParams +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdInNetworkQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird.ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer @@ -16,6 +27,7 @@ import com.twitter.product_mixer.core.functional_component.transformer.Candidate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @@ -24,7 +36,9 @@ import javax.inject.Singleton */ @Singleton class ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, + earlybirdTweetCandidateSource: EarlybirdRealtimeCGTweetCandidateSource, + followedUserScoresFeatureHydrator: FollowedUserScoresFeatureHydrator, + tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator, clientId: ClientId) extends CandidatePipelineConfig[ ScoredTweetsQuery, @@ -36,12 +50,24 @@ class ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig @Inject() ( override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsEarlybirdInNetwork") + override val supportedClientParam: Option[FSParam[Boolean]] = Some( + CandidateSourceParams.EnableInNetworkCandidateSourceParam) + override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) ) + override val queryFeatureHydration: Seq[ + BaseQueryFeatureHydrator[ScoredTweetsQuery, _] + ] = Seq( + ParamGatedQueryFeatureHydrator( + FeatureHydration.EnableFollowedUserScoreBackfillFeaturesParam, + followedUserScoresFeatureHydrator + ) + ) + override val candidateSource: BaseCandidateSource[eb.EarlybirdRequest, eb.ThriftSearchResult] = - earlybirdCandidateSource + earlybirdTweetCandidateSource override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, @@ -52,6 +78,19 @@ class ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig @Inject() ( CandidateFeatureTransformer[eb.ThriftSearchResult] ] = Seq(ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer) + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] + ] = Seq(tweetEntityServiceFeatureHydrator) + + override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( + QualifiedRepliesFilter, + ExtendedDirectedAtFilter + ) + + override val postFilterFeatureHydration: Seq[ + BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] + ] = Seq(IsExtendedReplyFeatureHydrator) + override val resultTransformer: CandidatePipelineResultsTransformer[ eb.ThriftSearchResult, TweetCandidate diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel index 4461d8460..65c871d31 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel @@ -4,10 +4,19 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/thrift/com/twitter/timelineservice:thrift-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "servo/repo/src/main/scala", + "src/thrift/com/twitter/search:earlybird-scala", + "src/thrift/com/twitter/timelines/pinned_timelines:thrift-scala", "stitch/stitch-timelineservice", + "strato/config/columns/content_understanding:content_understanding-strato-client", + "strato/config/columns/timelines/pintweet:pintweet-strato-client", + "strato/config/src/thrift/com/twitter/strato/columns/content_understanding:content_understanding-scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ContentExplorationCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ContentExplorationCandidateSource.scala new file mode 100644 index 000000000..826144b86 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ContentExplorationCandidateSource.scala @@ -0,0 +1,79 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_source + +import com.twitter.home_mixer.product.scored_tweets.query_transformer.ContentExplorationQueryRequest +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.stitch.Stitch +import com.twitter.strato.catalog.Scan.Slice +import com.twitter.strato.generated.client.content_understanding.CategoryColdStartPostsMhClientColumn +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Random + +case class ContentExplorationCandidateResponse( + tweetId: Long, + tier: String) + +@Singleton +class ContentExplorationCandidateSource @Inject() ( + coldStartPostsColumn: CategoryColdStartPostsMhClientColumn) + extends CandidateSource[ContentExplorationQueryRequest, ContentExplorationCandidateResponse] { + + private val MaxUserCategory = 5 + private val MaxResultPerCategory = 100 + private val MaxResult = 500 + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("ContentExploration") + + private val scanner = coldStartPostsColumn.scanner + + private def interweave( + candidates: Seq[Seq[ContentExplorationCandidateResponse]] + ): Seq[ContentExplorationCandidateResponse] = { + if (candidates.isEmpty) return Seq.empty + val maxLength = candidates.map(_.length).max + (0 until maxLength) + .flatMap { i => + candidates.flatMap(seq => seq.lift(i)) + }.take(MaxResult) + } + + private def getSlice(): Slice[Long] = { + val now = System.currentTimeMillis() + val startId = SnowflakeId.firstIdFor(now - 6 * 60 * 60 * 1000L) + val endId = SnowflakeId.firstIdFor(now) + + Slice[Long]( + from = Some(startId), + to = Some(endId), + limit = Some(MaxResultPerCategory) + ) + } + + override def apply( + request: ContentExplorationQueryRequest + ): Stitch[Seq[ContentExplorationCandidateResponse]] = { + val tags = request.userCategories + val scan = getSlice() + val version = request.version + + val scans: Seq[Stitch[Seq[ContentExplorationCandidateResponse]]] = tags.flatMap { tag => + Seq( + scanner.scan((version + "tier1_" + tag._1, scan)).map { results => + Random + .shuffle(results.map(_._1._2)) + .map(id => ContentExplorationCandidateResponse(id, "tier1")) + }, + scanner.scan((version + "tier2_" + tag._1, scan)).map { results => + Random + .shuffle(results.map(_._1._2)) + .map(id => ContentExplorationCandidateResponse(id, "tier2")) + } + ) + } + + Stitch.collect(scans).map(interweave) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/EarlybirdRealtimeCGTweetCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/EarlybirdRealtimeCGTweetCandidateSource.scala new file mode 100644 index 000000000..dab9098ff --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/EarlybirdRealtimeCGTweetCandidateSource.scala @@ -0,0 +1,27 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_source + +import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRealtimCGEndpoint +import com.twitter.product_mixer.component_library.candidate_source.earlybird.EarlybirdTweetCandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.search.earlybird.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class EarlybirdRealtimeCGTweetCandidateSource @Inject() ( + @Named(EarlybirdRealtimCGEndpoint) earlybirdService: t.EarlybirdService.MethodPerEndpoint) + extends EarlybirdTweetCandidateSource(earlybirdService) { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("EarlybirdRealtimeCGTweets") +} + +@Singleton +class EarlybirdNsfwCGTweetCandidateSource @Inject() ( + @Named(EarlybirdRealtimCGEndpoint) earlybirdService: t.EarlybirdService.MethodPerEndpoint) + extends EarlybirdTweetCandidateSource(earlybirdService) { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("EarlybirdNsfwCGTweets") +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala index 395ac7da8..7b31bbbff 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala @@ -5,23 +5,23 @@ import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIde import com.twitter.stitch.Stitch import com.twitter.stitch.timelineservice.TimelineService import com.twitter.timelineservice.{thriftscala => tls} - import javax.inject.Inject import javax.inject.Singleton +case class ListTweet(listId: Long, tweet: tls.Tweet) + @Singleton class ListsCandidateSource @Inject() (timelineService: TimelineService) - extends CandidateSource[Seq[tls.TimelineQuery], tls.Tweet] { + extends CandidateSource[Seq[tls.TimelineQuery], ListTweet] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("Lists") - override def apply(requests: Seq[tls.TimelineQuery]): Stitch[Seq[tls.Tweet]] = { - val timelines = Stitch.traverse(requests) { request => timelineService.getTimeline(request) } - - timelines.map { - _.flatMap { - _.entries.collect { case tls.TimelineEntry.Tweet(tweet) => tweet } + override def apply(requests: Seq[tls.TimelineQuery]): Stitch[Seq[ListTweet]] = Stitch + .traverse(requests) { request => + timelineService.getTimeline(request).map { response => + response.entries.collect { + case tls.TimelineEntry.Tweet(tweet) => ListTweet(response.timelineId.id, tweet) + } } - } - } + }.map(_.flatten) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/StaticPostsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/StaticPostsCandidateSource.scala new file mode 100644 index 000000000..20b4e1e64 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/StaticPostsCandidateSource.scala @@ -0,0 +1,57 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_source + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.servo.cache.ExpiringLruInProcessCache +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.timelines.pintweet.PinnedPostsClientColumn +import com.twitter.timelines.pinned_timelines.{thriftscala => pt} +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Random + +case class StaticSourceRequest(countryCodes: Seq[String], following: Set[Long]) + +object StaticPostsCandidateSource { + private val BaseTTL = 30 + private val StaticKey = 0 + private val TTL = (BaseTTL + Random.nextInt(15)).minutes + + val cache: InProcessCache[Int, Option[pt.PinnedPost]] = + new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 100) +} + +@Singleton +class StaticPostsCandidateSource @Inject() (pinnedPostsClientColumn: PinnedPostsClientColumn) + extends CandidateSource[StaticSourceRequest, Long] { + + import StaticPostsCandidateSource._ + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("StaticPosts") + + private val fetcher: Fetcher[Int, Unit, pt.PinnedPost] = pinnedPostsClientColumn.fetcher + + override def apply(request: StaticSourceRequest): Stitch[Seq[Long]] = { + val postsStitch = cache.get(StaticKey).map(Stitch.value(_)).getOrElse { + fetcher.fetch(StaticKey).map { response => + cache.set(StaticKey, response.v) + response.v + } + } + + postsStitch.map { + _.map { post => + val isInNetwork = request.following.contains(post.authorId) + val countryMatches = request.countryCodes.exists { code => + post.countryCodes.forall(_.map(_.toLowerCase).contains(code.toLowerCase)) + } + if (countryMatches && ((post.inNetwork && isInNetwork) || (post.outOfNetwork && !isInNetwork))) + Seq(post.postId) + else Seq.empty + }.getOrElse(Seq.empty) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel index 58a566de8..e67fbcb82 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel @@ -4,67 +4,33 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "representation-scorer/server/src/main/scala/com/twitter/representationscorer/common", "servo/repo/src/main/scala", - "servo/util/src/main/scala", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/java/com/twitter/ml/api/constant", - "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/adapters/conversation_features", - "src/scala/com/twitter/timelines/prediction/adapters/real_graph", - "src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph", - "src/scala/com/twitter/timelines/prediction/adapters/twistly", - "src/scala/com/twitter/timelines/prediction/adapters/two_hop_features", "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/common/util", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph", - "src/scala/com/twitter/timelines/prediction/features/time_features", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/onboarding/relevance/features:features-java", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", + "src/scala/com/twitter/timelines/prediction/features/semantic_core_features", "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "src/thrift/com/twitter/timelines/author_features:thrift-java", - "stitch/stitch-gizmoduck", - "stitch/stitch-socialgraph", - "stitch/stitch-tweetypie", - "strato/config/columns/audiencerewards/audienceRewardsService:getSuperFollowEligibility-strato-client", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/topics", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", - "timelines/src/main/scala/com/twitter/timelines/common/model", + "strato/config/columns/content_understanding:content_understanding-strato-client", + "strato/config/columns/lists/reads:core-strato-client", + "strato/config/columns/recommendations/user-signal-service:user-signal-service-strato-client", + "strato/config/columns/tweetypie/managed:managed-strato-client", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", "timelines/src/main/scala/com/twitter/timelines/model/types", - "topic-social-proof/server/src/main/thrift:thrift-scala", - "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", - "tweetconvosvc/thrift/src/main/thrift:thrift-scala", - "user_session_store/src/main/scala/com/twitter/user_session_store", + "tweetsource/common/src/main/thrift:thrift-scala", + "user-signal-service/thrift/src/main/thrift:thrift-scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala index 873f043be..5b498649b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala @@ -1,12 +1,15 @@ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.transport.Transport import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.param.HomeMixerInjectionNames.ScoredTweetsCache import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.TtlCache @@ -14,8 +17,8 @@ import com.twitter.stitch.Stitch import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Time - import javax.inject.Inject +import javax.inject.Named import javax.inject.Singleton /** @@ -23,14 +26,21 @@ import javax.inject.Singleton */ @Singleton case class CachedScoredTweetsQueryFeatureHydrator @Inject() ( + @Named(ScoredTweetsCache) scoredTweetsCache: TtlCache[Long, hmt.ScoredTweetsResponse]) - extends QueryFeatureHydrator[PipelineQuery] { + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("CachedScoredTweets") override val features: Set[Feature[_, _]] = Set(CachedScoredTweetsFeature) + override def onlyIf(query: PipelineQuery): Boolean = { + val serviceIdentifier = ServiceIdentifier.fromCertificate(Transport.peerCertificate) + serviceIdentifier.role != "explore-mixer" && serviceIdentifier.role != "video-mixer" + } + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId val tweetScoreTtl = query.params(CachedScoredTweets.TTLParam) @@ -42,7 +52,7 @@ case class CachedScoredTweetsQueryFeatureHydrator @Inject() ( val nonExpiredTweets = cachedScoredTweets.filter { tweet => tweet.lastScoredTimestampMs.exists(Time.fromMilliseconds(_).untilNow < tweetScoreTtl) } - FeatureMapBuilder().add(CachedScoredTweetsFeature, nonExpiredTweets).build() + FeatureMap(CachedScoredTweetsFeature, nonExpiredTweets) case Throw(exception) => throw exception } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala index cc3c2aeeb..dc95d2386 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala @@ -1,7 +1,7 @@ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.EarlybirdAdapter +import com.twitter.home_mixer.functional_component.feature_hydrator.UserLanguagesFeature import com.twitter.home_mixer.model.HomeFeatures.DeviceLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature @@ -9,9 +9,12 @@ import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.EarlybirdAdapter +import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.home_mixer.util.earlybird.EarlybirdResponseUtil import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.RichDataRecord import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature @@ -24,10 +27,13 @@ import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.search.common.constants.{thriftscala => scc} import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch +import com.twitter.timelines.earlybird.common.utils.InNetworkEngagement +import com.twitter.util.Future import com.twitter.util.Return import javax.inject.Inject import javax.inject.Named @@ -57,6 +63,7 @@ class EarlybirdFeatureHydrator @Inject() ( EarlybirdDataRecordFeature, EarlybirdFeature, EarlybirdSearchResultFeature, + SourceTweetEarlybirdFeature, TweetUrlsFeature ) @@ -64,88 +71,170 @@ class EarlybirdFeatureHydrator @Inject() ( private val scopedStatsReceiver = statsReceiver.scope(statScope) private val originalKeyFoundCounter = scopedStatsReceiver.counter("originalKey/found") - private val originalKeyLossCounter = scopedStatsReceiver.counter("originalKey/loss") + private val originalKeyNotFoundCounter = scopedStatsReceiver.counter("originalKey/notFound") - private val ebSearchResultNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean = - candidate => candidate.features.getOrElse(EarlybirdSearchResultFeature, None).isEmpty private val ebFeaturesNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean = candidate => candidate.features.getOrElse(EarlybirdFeature, None).isEmpty + private val InternalUrlPattern = "" + override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val candidatesToHydrate = candidates.filter { candidate => - val isEmpty = - ebFeaturesNotExistPredicate(candidate) && ebSearchResultNotExistPredicate(candidate) - if (isEmpty) originalKeyLossCounter.incr() else originalKeyFoundCounter.incr() - isEmpty - } + val candidatesWithoutExistingEBFeatures = candidates + .filter { candidate => + val isEmpty = ebFeaturesNotExistPredicate(candidate) + if (isEmpty) originalKeyNotFoundCounter.incr() else originalKeyFoundCounter.incr() + isEmpty + } + val (candidatesWithSearchFeatures, candidatesWithoutSearchFeatures) = + candidatesWithoutExistingEBFeatures.partition { candidate => + candidate.features.getOrElse(EarlybirdSearchResultFeature, None).nonEmpty + } + // hydrate both candidates and their sources + val candidateIdsToHydrate = ( + candidatesWithSearchFeatures + .filter(_.features.getOrElse(IsRetweetFeature, false)) + .map(CandidatesUtil.getOriginalTweetId) ++ + candidatesWithoutSearchFeatures.map(_.candidate.id) ++ + candidatesWithoutSearchFeatures.map(CandidatesUtil.getOriginalTweetId) + ).distinct + + client((candidateIdsToHydrate, query.getRequiredUserId)) + .flatMap( + handleResponse(query, candidates, _, candidateIdsToHydrate, candidatesWithSearchFeatures)) + } - client((candidatesToHydrate.map(_.candidate.id), query.getRequiredUserId)) - .map(handleResponse(query, candidates, _, candidatesToHydrate)) + private[feature_hydrator] def getFeatureMap( + candidate: CandidateWithFeatures[TweetCandidate], + query: PipelineQuery, + idToSearchResults: Map[Long, eb.ThriftSearchResult], + screenName: Option[String], + userLanguages: Seq[scc.ThriftLanguage], + uiLanguage: Option[scc.ThriftLanguage], + tweetCountByAuthorId: Map[Long, Int], + followedUserIds: Set[Long], + mutuallyFollowedUserIds: Set[Long], + inNetworkEngagement: InNetworkEngagement, + ): FeatureMap = { + val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false) + val existingEbFeatures = candidate.features.getOrElse(EarlybirdFeature, None) + val existingSourceTweetEbFeatures = + candidate.features.getOrElse(SourceTweetEarlybirdFeature, None) + + val tweetEbFeatures = + if (existingEbFeatures.nonEmpty) existingEbFeatures + else { + idToSearchResults.get(candidate.candidate.id).map { searchResult => + EarlybirdResponseUtil.getThriftTweetFeaturesFromSearchResult( + searcherUserId = query.getRequiredUserId, + screenName, + userLanguages, + uiLanguage, + tweetCountByAuthorId, + followedUserIds, + mutuallyFollowedUserIds, + idToSearchResults, + inNetworkEngagement, + searchResult + ) + } + } + + val sourceTweetEbFeatures = if (candidateIsRetweet) { + if (existingSourceTweetEbFeatures.nonEmpty) existingSourceTweetEbFeatures + else { + idToSearchResults.get(CandidatesUtil.getOriginalTweetId(candidate)).map { searchResult => + EarlybirdResponseUtil.getThriftTweetFeaturesFromSearchResult( + searcherUserId = query.getRequiredUserId, + screenName, + userLanguages, + uiLanguage, + tweetCountByAuthorId, + followedUserIds, + mutuallyFollowedUserIds, + idToSearchResults, + inNetworkEngagement, + searchResult + ) + } + } + } else None + + val originalTweetEbFeatures = + if (sourceTweetEbFeatures.nonEmpty) sourceTweetEbFeatures else tweetEbFeatures + val earlybirdDataRecord = + EarlybirdAdapter.adaptToDataRecords(originalTweetEbFeatures).asScala.head + + val tesUrls = candidate.features.getOrElse(TweetUrlsFeature, Seq.empty) + val urls = + if (tesUrls.isEmpty) tweetEbFeatures.flatMap(_.urlsList).getOrElse(Seq.empty) else tesUrls + + val rdr = new RichDataRecord(earlybirdDataRecord) + + FeatureMapBuilder(sizeHint = 5) + .add(EarlybirdFeature, tweetEbFeatures) + .add(EarlybirdDataRecordFeature, rdr.getRecord) + .add(EarlybirdSearchResultFeature, idToSearchResults.get(candidate.candidate.id)) + .add(SourceTweetEarlybirdFeature, sourceTweetEbFeatures) + .add(TweetUrlsFeature, urls) + .build() } private def handleResponse( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]], results: KeyValueResult[Long, eb.ThriftSearchResult], - candidatesToHydrate: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[FeatureMap] = { + candidateIdsToHydrate: Seq[Long], + candidatesWithSearchFeatures: Seq[CandidateWithFeatures[TweetCandidate]], + ): Future[Seq[FeatureMap]] = { + val batchSize = 64 val queryFeatureMap = query.features.getOrElse(FeatureMap.empty) val userLanguages = queryFeatureMap.getOrElse(UserLanguagesFeature, Seq.empty) val uiLanguageCode = queryFeatureMap.getOrElse(DeviceLanguageFeature, None) val screenName = queryFeatureMap.getOrElse(UserScreenNameFeature, None) val followedUserIds = queryFeatureMap.getOrElse(SGSFollowedUsersFeature, Seq.empty).toSet + val mutuallyFollowedUserIds = + queryFeatureMap.getOrElse(SGSMutuallyFollowedUsersFeature, Seq.empty).toSet - val searchResults = candidatesToHydrate - .map { candidate => - observedGet(Some(candidate.candidate.id), results) + val searchResults = candidateIdsToHydrate + .map { id => + observedGet(Some(id), results) }.collect { case Return(Some(value)) => value } - val allSearchResults = searchResults ++ - candidates.filter(!ebSearchResultNotExistPredicate(_)).flatMap { candidate => - candidate.features - .getOrElse(EarlybirdSearchResultFeature, None) - } - val idToSearchResults = allSearchResults.map(sr => sr.id -> sr).toMap - val tweetIdToEbFeatures = EarlybirdResponseUtil.getTweetThriftFeaturesByTweetId( - searcherUserId = query.getRequiredUserId, - screenName = screenName, - userLanguages = userLanguages, - uiLanguageCode = uiLanguageCode, - followedUserIds = followedUserIds, - mutuallyFollowingUserIds = Set.empty, - searchResults = allSearchResults, - sourceTweetSearchResults = Seq.empty, - ) - - candidates.map { candidate => - val transformedEbFeatures = tweetIdToEbFeatures.get(candidate.candidate.id) - val earlybirdFeatures = - if (transformedEbFeatures.nonEmpty) transformedEbFeatures - else candidate.features.getOrElse(EarlybirdFeature, None) - - val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false) - val sourceTweetEbFeatures = - candidate.features.getOrElse(SourceTweetEarlybirdFeature, None) - - val originalTweetEbFeatures = - if (candidateIsRetweet && sourceTweetEbFeatures.nonEmpty) - sourceTweetEbFeatures - else earlybirdFeatures - - val earlybirdDataRecord = - EarlybirdAdapter.adaptToDataRecords(originalTweetEbFeatures).asScala.head - - FeatureMapBuilder() - .add(EarlybirdFeature, earlybirdFeatures) - .add(EarlybirdDataRecordFeature, earlybirdDataRecord) - .add(EarlybirdSearchResultFeature, idToSearchResults.get(candidate.candidate.id)) - .add(TweetUrlsFeature, earlybirdFeatures.flatMap(_.urlsList).getOrElse(Seq.empty)) - .build() + val allSearchResults = searchResults ++ candidatesWithSearchFeatures.flatMap { candidate => + candidate.features.getOrElse(EarlybirdSearchResultFeature, None) } + + val idToSearchResults = allSearchResults.map { searchResults => + searchResults.id -> searchResults + }.toMap + + val uiLanguage = EarlybirdResponseUtil.getLanguage(uiLanguageCode) + val tweetCountByAuthorId = EarlybirdResponseUtil.getTweetCountByAuthorId(allSearchResults) + val inNetworkEngagement = + InNetworkEngagement(followedUserIds.toSeq, mutuallyFollowedUserIds, allSearchResults) + + OffloadFuturePools + .offloadBatchElementToElement[CandidateWithFeatures[TweetCandidate], FeatureMap]( + candidates, + candidate => + getFeatureMap( + candidate = candidate, + query = query, + idToSearchResults = idToSearchResults, + screenName = screenName, + userLanguages = userLanguages, + uiLanguage = uiLanguage, + tweetCountByAuthorId = tweetCountByAuthorId, + followedUserIds = followedUserIds, + mutuallyFollowedUserIds = mutuallyFollowedUserIds, + inNetworkEngagement = inNetworkEngagement + ), + batchSize + ) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FollowedUserScoresFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FollowedUserScoresFeatureHydrator.scala new file mode 100644 index 000000000..59bb3f987 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FollowedUserScoresFeatureHydrator.scala @@ -0,0 +1,65 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +object FollowedUserScoresFeature extends Feature[PipelineQuery, Map[Long, Double]] + +@Singleton +case class FollowedUserScoresFeatureHydrator @Inject() () + extends QueryFeatureHydrator[PipelineQuery] { + + private val UpdateFollowedUserThreshold = 1000 + private val MaxFollowedUsers = 1500 + private val DefaultAuthorScore = 2D + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("FollowedUserScores") + + override def features: Set[Feature[_, _]] = Set(FollowedUserScoresFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val sgsFollowedUserIds = + query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSeq.flatten + val sgsFollowedUserIdSet = sgsFollowedUserIds.toSet + val authorScoreMap = query.features + .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double])) + .getOrElse(Map.empty) + + val filteredAuthorScoreMap = authorScoreMap.filter { entry => + sgsFollowedUserIdSet.contains(entry._1) + } + + val updatedSgsFollowedUserIds = if (sgsFollowedUserIds.size >= UpdateFollowedUserThreshold) { + filteredAuthorScoreMap.keySet ++ + sgsFollowedUserIds + .filter(!filteredAuthorScoreMap.keySet.contains(_)) + .take(Math.max(0, MaxFollowedUsers - filteredAuthorScoreMap.size)) + } else sgsFollowedUserIds + + val existingAuthorIdScores = filteredAuthorScoreMap.values.toList.sorted + val imputedScoreIndex = + Math.min(existingAuthorIdScores.length - 1, (existingAuthorIdScores.length * 0.5f).toInt) + val imputedScore = + if (imputedScoreIndex >= 0) existingAuthorIdScores(imputedScoreIndex) + else DefaultAuthorScore + + val updatedAuthorScoreMap = updatedSgsFollowedUserIds + .map(_ -> imputedScore).toMap ++ filteredAuthorScoreMap + + Stitch.value { + FeatureMapBuilder() + .add(FollowedUserScoresFeature, updatedAuthorScoreMap) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/InvalidateCachedScoredTweetsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/InvalidateCachedScoredTweetsQueryFeatureHydrator.scala new file mode 100644 index 000000000..9e1634bfb --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/InvalidateCachedScoredTweetsQueryFeatureHydrator.scala @@ -0,0 +1,46 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature +import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature +import com.twitter.home_mixer.model.HomeFeatures.UserLastExplicitSignalTimeFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Invalidates (flushes) cached ScoredTweets upon detecting explicit signal from user made since + * previous request + */ +object InvalidateCachedScoredTweetsQueryFeatureHydrator + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "InvalidateCachedScoredTweets") + + override val features: Set[Feature[_, _]] = Set(CachedScoredTweetsFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val lastRequestTime = + query.features.getOrElse(FeatureMap.empty).getOrElse(LastNonPollingTimeFeature, None) + val lastExplicitSignalTime = + query.features + .getOrElse(FeatureMap.empty).getOrElse(UserLastExplicitSignalTimeFeature, None) + val hasRecentExplicitSignal = lastExplicitSignalTime + .flatMap { engagementTime => + lastRequestTime.map { requestTime => engagementTime > requestTime } + }.getOrElse(false) + + val featureMap = if (hasRecentExplicitSignal) { + FeatureMap(CachedScoredTweetsFeature, Seq.empty) + } else { + val cachedTweets = + query.features.getOrElse(FeatureMap.empty).getOrElse(CachedScoredTweetsFeature, Seq.empty) + FeatureMap(CachedScoredTweetsFeature, cachedTweets) + } + + Stitch.value(featureMap) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsColdStartPostFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsColdStartPostFeatureHydrator.scala new file mode 100644 index 000000000..609932cf8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsColdStartPostFeatureHydrator.scala @@ -0,0 +1,82 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.IsColdStartPostFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.IsColdStartPostInMemCache +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.cache.InProcessCache +import com.twitter.strato.generated.client.content_understanding.ColdStartPostsMetadataMhClientColumn +import com.twitter.stitch.Stitch +import com.twitter.strato.columns.content_understanding.content_exploration.thriftscala.ColdStartPostStatus +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class IsColdStartPostFeatureHydrator @Inject() ( + coldStartPostsMetadataMhClientColumn: ColdStartPostsMetadataMhClientColumn, + @Named(IsColdStartPostInMemCache) isColdStartPostInMemCache: InProcessCache[ + Long, + Boolean + ], + statsReceiver: StatsReceiver) + extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("IsColdStartPost") + + override val features: Set[Feature[_, _]] = Set(IsColdStartPostFeature) + + private val DefaultFeatureMap = FeatureMap(IsColdStartPostFeature, false) + + private val ineligibleStatusSet: Set[ColdStartPostStatus] = + Set(ColdStartPostStatus.Tier1Ineligible, ColdStartPostStatus.Tier1IneligibleHighQuality) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") + private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") + private val storeHitCounter = scopedStatsReceiver.counter("store/hit") + private val storeMissCounter = scopedStatsReceiver.counter("store/miss") + private val fetchExceptionCounter = scopedStatsReceiver.counter("fetch/exception") + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + existingFeatures: FeatureMap + ): Stitch[FeatureMap] = { + val postId = candidate.id + isColdStartPostInMemCache + .get(postId) + .map { cachedValue => + cacheHitCounter.incr() + Stitch.value(FeatureMap(IsColdStartPostFeature, cachedValue)) + }.getOrElse { + cacheMissCounter.incr() + coldStartPostsMetadataMhClientColumn.fetcher + .fetch(postId) + .map { response => + if (response.v.isDefined) { + storeHitCounter.incr() + } else { + storeMissCounter.incr() + } + val isColdStartPost = response.v.flatMap(_.status) match { + case Some(status) => !ineligibleStatusSet.contains(status) + case _ => false + } + isColdStartPostInMemCache.set(postId, isColdStartPost) + FeatureMap(IsColdStartPostFeature, isColdStartPost) + }.handle { + case _: Exception => + fetchExceptionCounter.incr() + DefaultFeatureMap + } + } + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListNameFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListNameFeatureHydrator.scala new file mode 100644 index 000000000..08e06002e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListNameFeatureHydrator.scala @@ -0,0 +1,48 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.ListIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ListNameFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.socialgraph.{thriftscala => sg} +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.lists.reads.CoreOnListClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ListNameFeatureHydrator @Inject() (coreOnListClientColumn: CoreOnListClientColumn) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ListName") + + override val features: Set[Feature[_, _]] = Set(ListNameFeature) + + private val fetcher: Fetcher[Long, Unit, sg.SocialgraphList] = coreOnListClientColumn.fetcher + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val listIds = candidates.flatMap(_.features.getOrElse(ListIdFeature, None)).distinct + + val listIdNameMapStitch = Stitch.collect { + listIds.map { listId => listId -> fetcher.fetch(listId).map(_.v.map(_.name)) }.toMap + } + + listIdNameMapStitch.map { listIdNameMap => + candidates.map { candidate => + val listId = candidate.features.getOrElse(ListIdFeature, None) + val listName = listId.flatMap(listIdNameMap.get).flatten + FeatureMapBuilder().add(ListNameFeature, listName).build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/LowSignalUserQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/LowSignalUserQueryFeatureHydrator.scala new file mode 100644 index 000000000..1566b176e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/LowSignalUserQueryFeatureHydrator.scala @@ -0,0 +1,90 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.LowSignalUserMaxSignalCount +import com.twitter.home_mixer.util.SignalUtil +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.recommendations.user_signal_service.SignalsClientColumn +import com.twitter.usersignalservice.{thriftscala => uss} +import com.twitter.util.Time +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class LowSignalUserQueryFeatureHydrator @Inject() ( + signalsClientColumn: SignalsClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("LowSignalUser") + + override val features: Set[Feature[_, _]] = + Set( + LowSignalUserFeature, + UserRecentEngagementTweetIdsFeature, + UserLastExplicitSignalTimeFeature) + + val fetcher: Fetcher[SignalsClientColumn.Key, Unit, SignalsClientColumn.Value] = + signalsClientColumn.fetcher + + val MaxFetch = 15L + val MinSignalFavCount = 0 + val MaxSignalFavCount = 5000000 + val LowSignalUserMaxSignalAge = 90.days + + private def getTimestamp(signal: uss.Signal): Option[Time] = { + if (signal.timestamp == 0L) None else Some(Time.fromMilliseconds(signal.timestamp)) + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val signalRequests = SignalUtil.ExplicitSignals.map { signal => + uss.SignalRequest( + maxResults = Some(MaxFetch), + signalType = signal, + minFavCount = Some(MinSignalFavCount), + maxFavCount = Some(MaxSignalFavCount) + ) + } + + val batchSignalRequest = uss.BatchSignalRequest( + userId = query.getRequiredUserId, + signalRequest = signalRequests, + clientId = Some(uss.ClientIdentifier.CrMixerHome) + ) + + fetcher.fetch(batchSignalRequest).map { response => + val signals = response.v.map(_.signalResponse.values.toSeq.flatten).getOrElse(Seq.empty) + val tweetIds = signals.collect { + case signal if signal.targetInternalId.isDefined => + signal.targetInternalId.get match { + case id: com.twitter.simclusters_v2.thriftscala.InternalId.TweetId => + Some(id.tweetId.toLong) + case _ => None + } + }.flatten + + val timeFilteredSignals = signals.filter { signal => + getTimestamp(signal).exists { signalTime => + Time.now.since(signalTime) < LowSignalUserMaxSignalAge + } + } + + val mostRecentTimestamp = + timeFilteredSignals.flatMap(getTimestamp).reduceOption((a, b) => if (a > b) a else b) + + val lowSignalUser = timeFilteredSignals.size < query.params(LowSignalUserMaxSignalCount) + FeatureMapBuilder() + .add(LowSignalUserFeature, lowSignalUser) + .add(UserRecentEngagementTweetIdsFeature, tweetIds.toList) + .add(UserLastExplicitSignalTimeFeature, mostRecentTimestamp) + .build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala index 80d857a26..ed513e3b2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala @@ -1,6 +1,8 @@ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieContentDataRecordFeature +import com.twitter.home_mixer.functional_component.feature_hydrator.WithDefaultFeatureMap import com.twitter.home_mixer.model.ContentFeatures import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.InReplyToContentFeatureAdapter @@ -12,7 +14,6 @@ import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier @@ -59,7 +60,8 @@ object InReplyToTweetypieContentDataRecordFeature */ @Singleton class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with WithDefaultFeatureMap { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ReplyTweet") @@ -70,14 +72,18 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) InReplyToTweetypieContentDataRecordFeature ) - private val defaulDataRecord: DataRecord = new DataRecord() + private val defaultDataRecord: DataRecord = new DataRecord() - private val DefaultFeatureMap = FeatureMapBuilder() - .add(ConversationDataRecordFeature, defaulDataRecord) - .add(InReplyToTweetHydratedEarlybirdFeature, None) - .add(InReplyToEarlybirdDataRecordFeature, defaulDataRecord) - .add(InReplyToTweetypieContentDataRecordFeature, defaulDataRecord) - .build() + override val defaultFeatureMap = FeatureMap( + ConversationDataRecordFeature, + defaultDataRecord, + InReplyToTweetHydratedEarlybirdFeature, + None, + InReplyToEarlybirdDataRecordFeature, + defaultDataRecord, + InReplyToTweetypieContentDataRecordFeature, + defaultDataRecord + ) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val hydratedReplyCounter = scopedStatsReceiver.counter("hydratedReply") @@ -87,14 +93,17 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { + // only hydrate for IN candidates + val eligibleCandidates = + candidates.filter(_.features.getOrElse(FromInNetworkSourceFeature, false)) val replyToInReplyToTweetMap = - ReplyRetweetUtil.replyTweetIdToInReplyToTweetMap(candidates) + ReplyRetweetUtil.replyTweetIdToInReplyToTweetMap(eligibleCandidates) val candidatesWithRepliesHydrated = candidates.map { candidate => replyToInReplyToTweetMap .get(candidate.candidate.id).map { inReplyToTweet => hydratedReplyCounter.incr() hydratedReplyCandidate(candidate, inReplyToTweet) - }.getOrElse((candidate, None, None)) + }.getOrElse((candidate, None, None, defaultDataRecord)) } /** @@ -102,12 +111,14 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) * the descendants. */ val ancestorTweetToDescendantRepliesMap = - ReplyRetweetUtil.ancestorTweetIdToDescendantRepliesMap(candidates) + ReplyRetweetUtil.ancestorTweetIdToDescendantRepliesMap(eligibleCandidates) val candidatesWithRepliesAndAncestorTweetsHydrated = candidatesWithRepliesHydrated.map { case ( maybeAncestorTweetCandidate, updatedReplyConversationFeatures, - inReplyToTweetEarlyBirdFeature) => + inReplyToTweetEarlyBirdFeature, + inReplyToTweetContentDataRecord + ) => ancestorTweetToDescendantRepliesMap .get(maybeAncestorTweetCandidate.candidate.id) .map { descendantReplies => @@ -120,35 +131,54 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) maybeAncestorTweetCandidate, descendantReplies, updatedReplyConversationFeatures) - (ancestorTweetCandidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) + ( + ancestorTweetCandidate, + inReplyToTweetEarlyBirdFeature, + updatedConversationFeatures, + inReplyToTweetContentDataRecord) } .getOrElse( ( maybeAncestorTweetCandidate, inReplyToTweetEarlyBirdFeature, - updatedReplyConversationFeatures)) + updatedReplyConversationFeatures, + inReplyToTweetContentDataRecord + )) } candidatesWithRepliesAndAncestorTweetsHydrated.map { - case (candidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) => + case ( + candidate, + inReplyToTweetEarlyBirdFeature, + updatedConversationFeatures, + inReplyToTweetContentDataRecord) => val conversationDataRecordFeature = updatedConversationFeatures .map(f => ConversationFeaturesAdapter.adaptToDataRecord(cf.ConversationFeatures.V1(f))) - .getOrElse(defaulDataRecord) + .getOrElse(defaultDataRecord) val inReplyToEarlybirdDataRecord = InReplyToEarlybirdAdapter .adaptToDataRecords(inReplyToTweetEarlyBirdFeature).asScala.head - val inReplyToContentDataRecord = InReplyToContentFeatureAdapter - .adaptToDataRecords( - inReplyToTweetEarlyBirdFeature.map(ContentFeatures.fromThrift)).asScala.head - - FeatureMapBuilder() - .add(ConversationDataRecordFeature, conversationDataRecordFeature) - .add(InReplyToTweetHydratedEarlybirdFeature, inReplyToTweetEarlyBirdFeature) - .add(InReplyToEarlybirdDataRecordFeature, inReplyToEarlybirdDataRecord) - .add(InReplyToTweetypieContentDataRecordFeature, inReplyToContentDataRecord) - .build() - case _ => DefaultFeatureMap + val inReplyToContentDataRecord = { + if (inReplyToTweetContentDataRecord.equals(defaultDataRecord)) { + InReplyToContentFeatureAdapter + .adaptToDataRecords( + inReplyToTweetEarlyBirdFeature.map(ContentFeatures.fromThrift)).asScala.head + } else + inReplyToTweetContentDataRecord + } + + FeatureMap( + ConversationDataRecordFeature, + conversationDataRecordFeature, + InReplyToTweetHydratedEarlybirdFeature, + inReplyToTweetEarlyBirdFeature, + InReplyToEarlybirdDataRecordFeature, + inReplyToEarlybirdDataRecord, + InReplyToTweetypieContentDataRecordFeature, + inReplyToContentDataRecord + ) + case _ => defaultFeatureMap } } @@ -158,7 +188,8 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) ): ( CandidateWithFeatures[TweetCandidate], Option[ConversationFeatures], - Option[ThriftTweetFeatures] + Option[ThriftTweetFeatures], + DataRecord ) = { val tweetedAfterInReplyToTweetInSecs = ( @@ -190,15 +221,24 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) // Note: if inReplyToTweet is a retweet, we need to read early bird feature from the merged // early bird feature field from RetweetSourceTweetFeatureHydrator class. // But if inReplyToTweet is a reply, we return its early bird feature directly + val sourceFeatures = inReplyToTweetCandidate.features + .getOrElse(SourceTweetEarlybirdFeature, None) val inReplyToTweetThriftTweetFeaturesOpt = { - if (inReplyToTweetCandidate.features.getOrElse(IsRetweetFeature, false)) { - inReplyToTweetCandidate.features.getOrElse(SourceTweetEarlybirdFeature, None) + if (inReplyToTweetCandidate.features.getOrElse(IsRetweetFeature, false) + && sourceFeatures.nonEmpty) { + sourceFeatures } else { inReplyToTweetCandidate.features.getOrElse(EarlybirdFeature, None) } } + val inReplyToTweetContentDataRecord = inReplyToTweetCandidate.features + .getOrElse(TweetypieContentDataRecordFeature, defaultDataRecord) - (replyCandidate, updatedConversationFeatures, inReplyToTweetThriftTweetFeaturesOpt) + ( + replyCandidate, + updatedConversationFeatures, + inReplyToTweetThriftTweetFeaturesOpt, + inReplyToTweetContentDataRecord) } private def hydrateAncestorTweetCandidate( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SGSMutuallyFollowedUserHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SGSMutuallyFollowedUserHydrator.scala new file mode 100644 index 000000000..e2b36b80f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SGSMutuallyFollowedUserHydrator.scala @@ -0,0 +1,58 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.socialgraph.{thriftscala => sg} +import com.twitter.stitch.Stitch +import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient} +import javax.inject.Inject +import javax.inject.Singleton + +object SGSMutuallyFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]] + +@Singleton +case class SGSMutuallyFollowedUserHydrator @Inject() ( + socialGraphStitchClient: SocialGraphStitchClient) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("SGSMutuallyFollowedUsers") + + override val features: Set[Feature[_, _]] = Set(SGSMutuallyFollowedUsersFeature) + + private val SocialGraphLimit = 14999 + private val MaxFollowTargets = 1500 + private val DefaultFeatureMap = FeatureMapBuilder() + .add(SGSMutuallyFollowedUsersFeature, Seq.empty) + .build() + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + + val sgsFollowedUserIds = + query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSeq.flatten + + if (sgsFollowedUserIds.nonEmpty) { + val mutuallyFollowedRequest = sg.IdsRequest( + relationships = Seq( + sg.SrcRelationship( + query.getRequiredUserId, + sg.RelationshipType.FollowedBy, + hasRelationship = true, + targets = Some(sgsFollowedUserIds.take(MaxFollowTargets)) + ), + ), + pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit))) + ) + socialGraphStitchClient.ids(mutuallyFollowedRequest).map(_.ids).map { mutuallyFollowedUsers => + FeatureMapBuilder() + .add(SGSMutuallyFollowedUsersFeature, mutuallyFollowedUsers) + .build() + } + } else Stitch.value(DefaultFeatureMap) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SemanticCoreFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SemanticCoreFeatureHydrator.scala new file mode 100644 index 000000000..922c3cfdb --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SemanticCoreFeatureHydrator.scala @@ -0,0 +1,112 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.home_mixer.functional_component.feature_hydrator.RealTimeEntityRealGraphFeatures +import com.twitter.home_mixer.functional_component.feature_hydrator.WithDefaultFeatureMap +import com.twitter.home_mixer.model.HomeFeatures.SemanticAnnotationIdsFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableRealTimeEntityRealGraphFeaturesParam +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.util.SRichDataRecord +import com.twitter.ml.api.{Feature => mlFeature} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.recos.entities.{thriftscala => ent} +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.features.semantic_core_features.SemanticCoreFeatures +import com.twitter.wtf.entity_real_graph.{thriftscala => erg} + +object SemanticCoreDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +object SemanticCoreFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] + with WithDefaultFeatureMap { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SemanticCore") + + override val features: Set[Feature[_, _]] = Set(SemanticCoreDataRecordFeature) + + override val defaultFeatureMap: FeatureMap = + FeatureMap(SemanticCoreDataRecordFeature, SemanticCoreDataRecordFeature.defaultValue) + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableRealTimeEntityRealGraphFeaturesParam) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { + val viewerErgFeatures = + query.features.flatMap(_.getOrElse(RealTimeEntityRealGraphFeatures, None)) + candidates.map { candidate => + val candidateEntities = candidate.features.getOrElse(SemanticAnnotationIdsFeature, Seq.empty) + + val engagements: Seq[Map[erg.EngagementType, erg.Features]] = { + viewerErgFeatures + .map { engagementsByEntity => + engagementsByEntity.collect { + case (ent.Entity.SemanticCore(sc), featuresByEngagementType) + if candidateEntities.contains(sc.entityId) => + featuresByEngagementType + }.toSeq + }.getOrElse(Seq.empty) + } + + val rawFeaturesByEngagementType: Map[erg.EngagementType, Seq[erg.Features]] = + SemanticCoreFeatures.engagementTypes.map { engagementType => + val features = engagements.flatMap(_.get(engagementType)) + engagementType -> features + }.toMap + + val decayedScoreFeatureValues: Map[mlFeature[_], List[Double]] = + SemanticCoreFeatures.decayedScoreFeatures.mapValues { engagementType => + val valuesOpt: Option[Seq[Double]] = + rawFeaturesByEngagementType.get(engagementType).map(_.map(_.scores.oneDayHalfLife)) + valuesOpt match { + case Some(values) => values.toList + case None => List.empty[Double] + } + } + val normalizedCountFeatureValues: Map[mlFeature[_], List[Double]] = + SemanticCoreFeatures.normalizedCountFeatures.mapValues { engagementType => + val valuesOpt: Option[Seq[Double]] = + rawFeaturesByEngagementType.get(engagementType).map(_.flatMap(_.normalizedCount)) + valuesOpt match { + case Some(values) => values.toList + case None => List.empty[Double] + } + } + + buildFeatureMap(decayedScoreFeatureValues ++ normalizedCountFeatureValues) + } + } + + private def buildFeatureMap( + featureValuesMap: Map[mlFeature[_], List[Double]] + ): FeatureMap = { + val featureContext = new FeatureContext(SemanticCoreFeatures.outputFeaturesPostMerge.toSeq: _*) + val richRecord = new SRichDataRecord(new DataRecord, featureContext) + + SemanticCoreFeatures.hydrateCountFeatures( + richRecord = richRecord, + features = SemanticCoreFeatures.precomputedCountFeatures, + featureValuesMap = featureValuesMap + ) + + FeatureMap(SemanticCoreDataRecordFeature, richRecord.getRecord) + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieVisibilityFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieVisibilityFeatureHydrator.scala new file mode 100644 index 000000000..6a2ecdfe6 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieVisibilityFeatureHydrator.scala @@ -0,0 +1,255 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature +import com.twitter.home_mixer.model.HomeFeatures.OonNsfwFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetEntityServiceVisibilityMigrationParam +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.spam.rtf.{thriftscala => rtf} +import com.twitter.stitch.Stitch +import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} +import com.twitter.strato.client.Client +import com.twitter.strato.generated.client.tweetypie.managed.HomeMixerOnTweetClientColumn +import com.twitter.tweetypie.{thriftscala => tp} +import com.twitter.util.logging.Logging +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +case class TweetypieVisibilityFeatures( + communityId: Option[Long], + isHydrated: Boolean, + isNsfw: Boolean, + oonNsfw: Boolean, + tweetText: Option[String], + tweetLanguage: Option[String], + visibilityReason: Option[rtf.FilteredReason]) + +@Singleton +class TweetypieVisibilityFeatureHydrator @Inject() ( + tweetypieStitchClient: TweetypieStitchClient, + statsReceiver: StatsReceiver, + @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Logging { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TweetypieVisibility") + + private val tweetypieTweetsVisibilityFoundCounter = + statsReceiver.counter("VisibilityFeatureTweetypieTweetsFound") + private val tweetypieTweetsVisibilityNotFoundCounter = + statsReceiver.counter("VisibilityFeatureTweetypieTweetsNotFound") + private val tesTweetsVisibilityFoundCounter = + statsReceiver.counter("VisibilityFeatureTesTweetsFound") + private val tesTweetsVisibilityNotFoundCounter = + statsReceiver.counter("VisibilityFeatureTesTweetsNotFound") + + override val features: Set[Feature[_, _]] = Set( + CommunityIdFeature, + IsHydratedFeature, + IsNsfw, + OonNsfwFeature, + TweetTextFeature, + TweetLanguageFeature, + VisibilityReason + ) + + private val HydrationFields: Set[tp.TweetInclude] = Set( + tp.TweetInclude.TweetFieldId(tp.Tweet.CommunitiesField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.CoreDataField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.IdField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.LanguageField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.QuotedTweetField.id) + ) + + private val DefaultTweetFieldsOptions = tp.GetTweetFieldsOptions( + tweetIncludes = HydrationFields, + includeRetweetedTweet = true, + includeQuotedTweet = true, + visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, + safetyLevel = Some(rtf.SafetyLevel.TimelineHome) + ) + + private val OutOfNetworkTweetFieldsOptions = + DefaultTweetFieldsOptions.copy(safetyLevel = Some(rtf.SafetyLevel.TimelineHomeRecommendations)) + + private val DefaultTweetypieVisibilityFeatures = TweetypieVisibilityFeatures( + communityId = None, + isHydrated = false, + isNsfw = false, + oonNsfw = false, + tweetLanguage = None, + tweetText = None, + visibilityReason = None + ) + + private def buildFeatureMap( + gtfResult: Stitch[tp.GetTweetFieldsResult], + inNetwork: Boolean, + fromTes: Boolean, + tweetId: Long + ): Stitch[(Long, TweetypieVisibilityFeatures)] = { + gtfResult.map { + case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quote, _) => + if (fromTes) tesTweetsVisibilityFoundCounter.incr() + else tweetypieTweetsVisibilityFoundCounter.incr() + + val coreData = found.tweet.coreData + + val isNsfwAdmin = coreData.exists(_.nsfwAdmin) + val isNsfwUser = coreData.exists(_.nsfwUser) + val sourceTweetIsNsfw = + found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser)) + + val quotedTweetDropped = quote.exists { + case _: tp.TweetFieldsResultState.Filtered => true + case _: tp.TweetFieldsResultState.NotFound => true + case _ => false + } + val quotedTweetIsNsfw = quote.exists { + case quoteTweet: tp.TweetFieldsResultState.Found => + quoteTweet.found.tweet.coreData.exists(data => data.nsfwAdmin || data.nsfwUser) + case _ => false + } + + val isNsfw = isNsfwAdmin || isNsfwUser || sourceTweetIsNsfw || quotedTweetIsNsfw + + val communityId = found.tweet.communities.flatMap(_.communityIds.headOption) + + tweetId -> TweetypieVisibilityFeatures( + communityId = communityId, + // Since this tweet was Found, it is not dropped, so we only need to check if there + // was a dropped quoted tweet. + isHydrated = !quotedTweetDropped, + isNsfw = isNsfw, + oonNsfw = !inNetwork && isNsfw, + tweetLanguage = found.tweet.language.map(_.language), + tweetText = coreData.map(_.text), + visibilityReason = found.suppressReason + ) + + case _ => + if (fromTes) tesTweetsVisibilityNotFoundCounter.incr() + else tweetypieTweetsVisibilityNotFoundCounter.incr() + tweetId -> DefaultTweetypieVisibilityFeatures + } + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + + try { + val followedUserIds = query.features.get.getOrElse(SGSFollowedUsersFeature, Seq.empty).toSet + val inNetworkTweetFieldsOptions = + DefaultTweetFieldsOptions.copy(forUserId = Some(query.getRequiredUserId)) + + val outOfNetworkTweetFieldsOptions = + OutOfNetworkTweetFieldsOptions.copy(forUserId = Some(query.getRequiredUserId)) + + val resultsStitch = Stitch + .collect { + candidates + .flatMap { candidate => + val ancestors = candidate.features.getOrElse(AncestorsFeature, Seq.empty).map { + ancestor => + val inNetwork = + ancestor.userId == query.getRequiredUserId || + followedUserIds.contains(ancestor.userId) + (ancestor.tweetId, ancestor.userId, inNetwork) + } + + val authorId = candidate.features.get(AuthorIdFeature).get + val inNetwork = + authorId == query.getRequiredUserId || followedUserIds.contains(authorId) + + Seq((candidate.candidate.id, authorId, inNetwork)) ++ ancestors.headOption ++ + ancestors.lastOption + }.distinct.map { + case (tweetId, _, inNetwork) => + val gtfOptions = + if (inNetwork) inNetworkTweetFieldsOptions else outOfNetworkTweetFieldsOptions + + try { + if (query.params(EnableTweetEntityServiceVisibilityMigrationParam)) { + val fetcher = new HomeMixerOnTweetClientColumn(stratoClient).fetcher + val response = fetcher.fetch(tweetId, gtfOptions).map(_.v) + response.flatMap { + case Some(result) => + buildFeatureMap(Stitch.value(result), inNetwork, fromTes = true, tweetId) + case None => + tesTweetsVisibilityNotFoundCounter.incr() + Stitch.value(tweetId -> DefaultTweetypieVisibilityFeatures) + } + } else { + buildFeatureMap( + tweetypieStitchClient.getTweetFields(tweetId, gtfOptions), + inNetwork, + fromTes = false, + tweetId + ) + } + } catch { + case e: Exception => + error(s"2 - Error fetching tweetypie visibility: $e") + Stitch.value(tweetId -> DefaultTweetypieVisibilityFeatures) + } + } + }.onFailure { + case e: Exception => + error(s"1 - Error fetching tweetypie visibility: $e") + } + + resultsStitch.map { results => + val resultsMap = results.toMap + candidates.map { candidate => + val ancestors = candidate.features.getOrElse(AncestorsFeature, Seq.empty) + val ancestorTweetypieVisibilityFeatures = + (ancestors.headOption ++ ancestors.lastOption).toSeq.distinct.map { ancestor => + resultsMap.getOrElse(ancestor.tweetId, DefaultTweetypieVisibilityFeatures) + } + + val ancestorsHydrated = + ancestorTweetypieVisibilityFeatures.map(_.isHydrated).forall(identity) + val ancestorsOonNsfw = ancestorTweetypieVisibilityFeatures.map(_.oonNsfw).exists(identity) + val ancestorsNsfw = ancestorTweetypieVisibilityFeatures.map(_.isNsfw).exists(identity) + + val tweetypieVisibilityFeatures = + resultsMap.getOrElse(candidate.candidate.id, DefaultTweetypieVisibilityFeatures) + + FeatureMapBuilder() + .add(CommunityIdFeature, tweetypieVisibilityFeatures.communityId) + .add(IsHydratedFeature, tweetypieVisibilityFeatures.isHydrated && ancestorsHydrated) + .add(IsNsfw, Some(tweetypieVisibilityFeatures.isNsfw || ancestorsNsfw)) + .add(OonNsfwFeature, tweetypieVisibilityFeatures.oonNsfw || ancestorsOonNsfw) + .add(TweetLanguageFeature, tweetypieVisibilityFeatures.tweetLanguage) + .add(TweetTextFeature, tweetypieVisibilityFeatures.tweetText) + .add(VisibilityReason, tweetypieVisibilityFeatures.visibilityReason) + .build() + } + } + } catch { + case e: Exception => + error(s"3 - Error fetching tweetypie visibility: $e") + Stitch.value(Seq.empty) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ValidLikedByUserIdsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ValidLikedByUserIdsFeatureHydrator.scala new file mode 100644 index 000000000..9afca0fb1 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ValidLikedByUserIdsFeatureHydrator.scala @@ -0,0 +1,34 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.ValidLikedByUserIdsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object ValidLikedByUserIdsFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ValidLikedByUserIds") + + override val features: Set[Feature[_, _]] = Set(ValidLikedByUserIdsFeature) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + Stitch.value { + candidates.map { candidate => + val validLikers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) + FeatureMapBuilder().add(ValidLikedByUserIdsFeature, validLikers).build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel index 6a9393121..a9beaa31b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel @@ -6,13 +6,8 @@ scala_library( dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/features/common", - "src/scala/com/twitter/timelines/prediction/features/conversation_features", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel index 9428c0d39..2f294b929 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel @@ -5,15 +5,11 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/features/common", "src/scala/com/twitter/timelines/prediction/features/recap", "src/scala/com/twitter/timelines/util", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", "src/thrift/com/twitter/search/common:features-scala", - "timelines/src/main/scala/com/twitter/timelines/util", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala index 5a0207e8a..275cc1191 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala @@ -33,7 +33,7 @@ object EarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTwe RecapFeatures.HAS_CARD, RecapFeatures.HAS_CONSUMER_VIDEO, RecapFeatures.HAS_HASHTAG, - RecapFeatures.HAS_IMAGE, +// RecapFeatures.HAS_IMAGE, RecapFeatures.HAS_LINK, RecapFeatures.HAS_MENTION, RecapFeatures.HAS_MULTIPLE_HASHTAGS_OR_TRENDS, @@ -44,7 +44,7 @@ object EarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTwe RecapFeatures.HAS_PERISCOPE, RecapFeatures.HAS_PRO_VIDEO, RecapFeatures.HAS_TREND, - RecapFeatures.HAS_VIDEO, +// RecapFeatures.HAS_VIDEO, RecapFeatures.HAS_VINE, RecapFeatures.HAS_VISIBLE_LINK, RecapFeatures.IS_AUTHOR_BOT, @@ -196,9 +196,9 @@ object EarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTwe richDataRecord .setFeatureValue[JBoolean](RecapFeatures.HAS_NATIVE_IMAGE, features.hasNativeImage) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_CARD, features.hasCard) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_IMAGE, features.hasImage) +// richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_IMAGE, features.hasImage) //handled via TweetEntityServiceContentFeatureHydrator richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_NEWS, features.hasNews) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_VIDEO, features.hasVideo) +// richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_VIDEO, features.hasVideo) //handled via TweetEntityServiceContentFeatureHydrator richDataRecord.setFeatureValue[JBoolean](RecapFeatures.CONTAINS_MEDIA, features.containsMedia) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.RETWEET_SEARCHER, features.retweetSearcher) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/BUILD.bazel index 07a2e7cb3..ed7aad8f9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/BUILD.bazel @@ -4,11 +4,21 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/control_ai", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "stitch/stitch-core", + "snowflake/src/main/scala/com/twitter/snowflake/id", + "src/thrift/com/twitter/timelines/control_ai:timeline-control-ai-thrift-scala", + "stitch/stitch-socialgraph", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ControlAiExcludeFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ControlAiExcludeFilter.scala new file mode 100644 index 000000000..d205b501d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ControlAiExcludeFilter.scala @@ -0,0 +1,42 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ControlAiEmbeddingSimilarityThresholdParam +import com.twitter.home_mixer.product.scored_tweets.util.ControlAiUtil +import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.ControlAiTopicEmbeddingMapFeature +import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.UserControlAiFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelines.control_ai.control.{thriftscala => ci} + +object ControlAiExcludeFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("ControlAiExclude") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val actions = query.features + .flatMap(_.getOrElse(UserControlAiFeature, None)) + .map(_.actions).getOrElse(Seq.empty).filter(_.actionType == ci.ActionType.Exclude) + + val (removed, kept) = candidates.partition { candidate => + actions.exists( + ControlAiUtil.conditionMatch( + _, + candidate, + query.features + .map(_.get(ControlAiTopicEmbeddingMapFeature)).getOrElse(Map.empty), + threshold = query.params(ControlAiEmbeddingSimilarityThresholdParam) + )) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ControlAiOnlyIncludeFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ControlAiOnlyIncludeFilter.scala new file mode 100644 index 000000000..a5ae1cea6 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ControlAiOnlyIncludeFilter.scala @@ -0,0 +1,42 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ControlAiEmbeddingSimilarityThresholdParam +import com.twitter.home_mixer.product.scored_tweets.util.ControlAiUtil +import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.ControlAiTopicEmbeddingMapFeature +import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.UserControlAiFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelines.control_ai.control.{thriftscala => ci} + +object ControlAiOnlyIncludeFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("ControlAiOnlyInclude") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val actions = query.features + .flatMap(_.getOrElse(UserControlAiFeature, None)) + .map(_.actions).getOrElse(Seq.empty).filter(_.actionType == ci.ActionType.Only) + + val (kept, removed) = candidates.partition { candidate => + actions.isEmpty || actions.exists( + ControlAiUtil.conditionMatch( + _, + candidate, + query.features + .map(_.get(ControlAiTopicEmbeddingMapFeature)).getOrElse(Map.empty), + threshold = query.params(ControlAiEmbeddingSimilarityThresholdParam) + )) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilter.scala new file mode 100644 index 000000000..36d1cc48f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilter.scala @@ -0,0 +1,49 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.UniversalNoun +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.stitch.Stitch +import com.twitter.timelines.configapi.Param +import com.twitter.util.Duration + +/** + * Filters Snowflake ID-compatible content that are older than some configurable threshold. + * + * We derive the content age from the Snowflake ID (http://go/snowflake), and non-Snowflake IDs + * are assumed to be too old. + * + * @param maxAgeParam Feature Switch configurable for convenience + * @tparam Candidate The type of the candidates + */ +case class CustomSnowflakeIdAgeFilter[Candidate <: UniversalNoun[Long]]( + maxAgeParam: Param[Duration]) + extends Filter[PipelineQuery, Candidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("SnowflakeIdAge") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[Candidate]] + ): Stitch[FilterResult[Candidate]] = { + val maxAge = query.params(maxAgeParam) + + val (keptCandidates, removedCandidates) = candidates + .map(c => (c.candidate, c.features.get(CandidatePipelines))) + .partition { filterCandidate => + SnowflakeId.timeFromIdOpt(filterCandidate._1.id) match { + case Some(creationTime) => + query.queryTime.since(creationTime) <= maxAge + case _ => false // Always deny if non-Snowflake + } + } + + Stitch.value( + FilterResult(kept = keptCandidates.map(_._1), removed = removedCandidates.map(_._1))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilterWithIncludeRule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilterWithIncludeRule.scala new file mode 100644 index 000000000..60741f0df --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilterWithIncludeRule.scala @@ -0,0 +1,56 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.UniversalNoun +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.stitch.Stitch +import com.twitter.timelines.configapi.Param +import com.twitter.util.Duration + +/** + * Filters Snowflake ID-compatible content that are older than some configurable threshold, + * but only for those candidates that match the includeRule. + * + * Candidates not matching the includeRule will always be kept. + * + * @param maxAgeParam Feature Switch configurable for convenience + * @tparam Candidate The type of the candidates + */ +case class CustomSnowflakeIdAgeFilterWithIncludeRule[Candidate <: UniversalNoun[Long]]( + maxAgeParam: Param[Duration], + includeRule: Option[CandidateWithFeatures[Candidate] => Boolean] = None) + extends Filter[PipelineQuery, Candidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("SnowflakeIdAgeWithIncludeRule") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[Candidate]] + ): Stitch[FilterResult[Candidate]] = { + val maxAge = query.params(maxAgeParam) + + val (keptCandidates, removedCandidates) = candidates.partition { filterCandidate => + includeRule match { + case Some(rule) if rule(filterCandidate) => + SnowflakeId.timeFromIdOpt(filterCandidate.candidate.id) match { + case Some(creationTime) => + query.queryTime.since(creationTime) <= maxAge + case _ => false + } + case _ => + true + } + } + + Stitch.value( + FilterResult( + kept = keptCandidates.map(_.candidate), + removed = removedCandidates.map(_.candidate) + ) + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilterWithSkipRule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilterWithSkipRule.scala new file mode 100644 index 000000000..831b40a69 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilterWithSkipRule.scala @@ -0,0 +1,54 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.UniversalNoun +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.stitch.Stitch +import com.twitter.timelines.configapi.Param +import com.twitter.util.Duration + +/** + * Filters Snowflake ID-compatible content that are older than some configurable threshold. + * + * We derive the content age from the Snowflake ID (http://go/snowflake), and non-Snowflake IDs + * are assumed to be too old. + * + * @param maxAgeParam Feature Switch configurable for convenience + * @tparam Candidate The type of the candidates + */ +case class CustomSnowflakeIdAgeFilterWithSkipRule[Candidate <: UniversalNoun[Long]]( + maxAgeParam: Param[Duration], + skipRule: Option[CandidateWithFeatures[Candidate] => Boolean] = None) + extends Filter[PipelineQuery, Candidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("SnowflakeIdAgeWithSkipRule") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[Candidate]] + ): Stitch[FilterResult[Candidate]] = { + val maxAge = query.params(maxAgeParam) + + val (keptCandidates, removedCandidates) = candidates.partition { filterCandidate => + skipRule match { + case Some(rule) if rule(filterCandidate) => true + case _ => + SnowflakeId.timeFromIdOpt(filterCandidate.candidate.id) match { + case Some(creationTime) => query.queryTime.since(creationTime) <= maxAge + case _ => false + } + } + } + + Stitch.value( + FilterResult( + kept = keptCandidates.map(_.candidate), + removed = removedCandidates.map(_.candidate) + ) + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/DuplicateConversationTweetsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/DuplicateConversationTweetsFilter.scala index adc11d255..5d73ba2df 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/DuplicateConversationTweetsFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/DuplicateConversationTweetsFilter.scala @@ -1,6 +1,7 @@ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter @@ -12,6 +13,7 @@ import com.twitter.stitch.Stitch /** * Remove any candidate that is in the ancestor list of any reply, including retweets of ancestors. + * Then deduplicate any replies with the same root tweet by score. * * E.g. if B replied to A and D was a retweet of A, we would prefer to drop D since otherwise * we may end up serving the same tweet twice in the timeline (e.g. serving both A->B and D). @@ -28,9 +30,17 @@ object DuplicateConversationTweetsFilter extends Filter[PipelineQuery, TweetCand .flatMap(_.features.getOrElse(AncestorsFeature, Seq.empty)) .map(_.tweetId).toSet - val (kept, removed) = candidates.partition { candidate => - !allAncestors.contains(CandidatesUtil.getOriginalTweetId(candidate)) - } + val dedupedCandidates = candidates + .filter(candidate => !allAncestors.contains(CandidatesUtil.getOriginalTweetId(candidate))) + .groupBy(_.features.getOrElse(AncestorsFeature, Seq.empty).lastOption.map(_.tweetId)) + .flatMap { + case (Some(_), conversationCandidates) => + Seq(conversationCandidates.maxBy(_.features.getOrElse(ScoreFeature, None)).candidate.id) + case (None, nonConversationCandidates) => nonConversationCandidates.map(_.candidate.id) + }.toSet + + val (kept, removed) = + candidates.partition(candidate => dedupedCandidates.contains(candidate.candidate.id)) Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ExtendedDirectedAtFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ExtendedDirectedAtFilter.scala new file mode 100644 index 000000000..76dbd1a13 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ExtendedDirectedAtFilter.scala @@ -0,0 +1,36 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object ExtendedDirectedAtFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("ExtendedDirectedAt") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val sgsFollowedUsers = + query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSet.flatten + + val (removed, kept) = candidates.partition { candidate => + val authorId = candidate.features.getOrElse(AuthorIdFeature, None) + val inReplyToUser = candidate.features.getOrElse(InReplyToUserIdFeature, None) + val directedAtUser = candidate.features.getOrElse(DirectedAtUserIdFeature, None) + + inReplyToUser.isEmpty && directedAtUser.exists(!sgsFollowedUsers.contains(_)) && + authorId.exists(sgsFollowedUsers.contains) + } + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/FollowedAuthorFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/FollowedAuthorFilter.scala new file mode 100644 index 000000000..e92b4cca8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/FollowedAuthorFilter.scala @@ -0,0 +1,31 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * This filter removes tweets who's author is in the follow list + */ +object FollowedAuthorFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("FollowedAuthor") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val sgsFollowedUsers = + query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSet.flatten + val (removed, kept) = candidates.partition( + _.features.getOrElse(AuthorIdFeature, None).exists(sgsFollowedUsers.contains)) + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/GrokAutoTranslateLanguageFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/GrokAutoTranslateLanguageFilter.scala new file mode 100644 index 000000000..55be701d2 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/GrokAutoTranslateLanguageFilter.scala @@ -0,0 +1,64 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.model.HomeFeatures.GrokTranslatedPostIsCachedFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFromTweetypieFeature +import com.twitter.home_mixer.model.HomeFeatures.UserUnderstandableLanguagesFeature +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableGrokAutoTranslateLanguageFilter +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.stitch.Stitch + +object GrokAutoTranslateLanguageFilter + extends Filter[ScoredTweetsQuery, TweetCandidate] + with Filter.Conditionally[ScoredTweetsQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("GrokAutoTranslateLanguage") + + override def onlyIf( + query: ScoredTweetsQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableGrokAutoTranslateLanguageFilter) + + override def apply( + query: ScoredTweetsQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val allUserUnderstandableLanguages: Seq[String] = + query.features + .getOrElse(FeatureMap.empty).getOrElse( + UserUnderstandableLanguagesFeature, + Seq.empty[String]) + + val (kept, removed) = { + if (!allUserUnderstandableLanguages.isEmpty) { + candidates.partition { candidate => + val inNetwork = + candidate.features.getOrElse(InNetworkFeature, true) + val postLanguageOpt = + candidate.features.getOrElse(TweetLanguageFromTweetypieFeature, None).map(_.toLowerCase) + val grokTranslateCacheExists = + candidate.features.getOrElse(GrokTranslatedPostIsCachedFeature, true) + + postLanguageOpt.forall { postLanguage => + inNetwork || grokTranslateCacheExists || + allUserUnderstandableLanguages.contains(postLanguage) + } + } + } else { (candidates, Seq.empty) } + } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/IsOutOfNetworkColdStartPostFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/IsOutOfNetworkColdStartPostFilter.scala new file mode 100644 index 000000000..c47b4b822 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/IsOutOfNetworkColdStartPostFilter.scala @@ -0,0 +1,34 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.IsColdStartPostFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object IsOutOfNetworkColdStartPostFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("IsOutOfNetworkColdStartPost") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (removed, kept) = candidates.partition { candidate => + val isColdStartPost = + candidate.features.getOrElse( + IsColdStartPostFeature, + false + ) // Id none, assume its not cold start, don't filter out + val isOutOfNetwork = + !candidate.features + .getOrElse(InNetworkFeature, true) // If none, assume its in-network, don't filter out + isColdStartPost && isOutOfNetwork + } + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/LanguageFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/LanguageFilter.scala new file mode 100644 index 000000000..bc36edc7e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/LanguageFilter.scala @@ -0,0 +1,69 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.functional_component.feature_hydrator.UserLanguagesFeature +import com.twitter.home_mixer.model.HomeFeatures.DeviceLanguageFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFromTweetypieFeature +import com.twitter.home_mixer.model.HomeFeatures.UserEngagedLanguagesFeature +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableLanguageFilter +import com.twitter.home_mixer.util.LanguageCode.AllowedLanguageCodes +import com.twitter.home_mixer.util.LanguageCode.languageToISO +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.stitch.Stitch + +object LanguageFilter + extends Filter[ScoredTweetsQuery, TweetCandidate] + with Filter.Conditionally[ScoredTweetsQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("Language") + + override def onlyIf( + query: ScoredTweetsQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableLanguageFilter) + + override def apply( + query: ScoredTweetsQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val userLanguages = query.features.get + .getOrElse(UserLanguagesFeature, Seq.empty) + .flatMap(lang => languageToISO.get(lang.toString.toLowerCase)) + .toSet + + val userEngagedLanguages = query.features.get + .getOrElse(UserEngagedLanguagesFeature, Set.empty[String]) + .map(_.toLowerCase) + + val deviceLanguage = + query.features.get.getOrElse(DeviceLanguageFeature, None).map(_.toLowerCase).toSet + + val allUserLanguages = userLanguages ++ userEngagedLanguages ++ deviceLanguage + + val (kept, removed) = if (!allUserLanguages.isEmpty) { + candidates.partition { candidate => + val inNetwork = candidate.features.getOrElse(InNetworkFeature, true) + + val postLanguage = candidate.features + .getOrElse(TweetLanguageFromTweetypieFeature, None) + .map(_.toLowerCase) + + postLanguage.forall { lang => + inNetwork || allUserLanguages.contains(lang) || AllowedLanguageCodes.contains(lang) + } + } + } else { (candidates, Seq.empty) } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OONReplyFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OONReplyFilter.scala new file mode 100644 index 000000000..8663b071c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OONReplyFilter.scala @@ -0,0 +1,43 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * This filter removes recommended replies to not followed users + */ +object OONReplyFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("OONReply") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val sgsFollowedUsers = + query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSet.flatten + val (removed, kept) = candidates.partition { candidate => + val isValidRecommendedReply = + !candidate.features.getOrElse(IsRetweetFeature, false) && + candidate.features + .getOrElse(InReplyToUserIdFeature, None).exists(sgsFollowedUsers.contains) + + val isRecommendedReply = + candidate.features.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && + !candidate.features.getOrElse(AuthorIdFeature, None).exists(sgsFollowedUsers.contains) + + isRecommendedReply && !isValidRecommendedReply + } + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/QualifiedRepliesFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/QualifiedRepliesFilter.scala new file mode 100644 index 000000000..321df6a18 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/QualifiedRepliesFilter.scala @@ -0,0 +1,45 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsInReplyToReplyOrDirectedFeature +import com.twitter.home_mixer.model.HomeFeatures.IsInReplyToRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object QualifiedRepliesFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("QualifiedReplies") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val sgsFollowedUsers = + query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSet.flatten + + val (removed, kept) = candidates.partition { candidate => + val isRetweet = candidate.features.getOrElse(IsRetweetFeature, false) + val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) + val inReplyToUser = candidate.features.getOrElse(InReplyToUserIdFeature, None) + val replyToFollowed = inReplyToUser.exists(sgsFollowedUsers.contains) + val isValidReplyToUser = inReplyToUser.exists { user => + user != query.getRequiredUserId && user != authorId + } + val inReplyToRetweetOrReplyOrDirected = + candidate.features.getOrElse(IsInReplyToReplyOrDirectedFeature, false) || + candidate.features.getOrElse(IsInReplyToRetweetFeature, false) + + inReplyToUser.nonEmpty && !replyToFollowed && sgsFollowedUsers.contains(authorId) && + (isRetweet || !isValidReplyToUser || inReplyToRetweetOrReplyOrDirected) + } + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/SGSAuthorFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/SGSAuthorFilter.scala new file mode 100644 index 000000000..bd3336377 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/SGSAuthorFilter.scala @@ -0,0 +1,60 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.socialgraph.{thriftscala => sg} +import com.twitter.stitch.Stitch +import com.twitter.stitch.socialgraph.SocialGraph +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class SGSAuthorFilter @Inject() (socialGraph: SocialGraph) + extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("SGSAuthor") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val authorIds = + candidates.flatMap { candidate => candidate.features.get(AuthorIdFeature) }.distinct + + val request = sg.IdsRequest( + relationships = Seq( + sg.SrcRelationship( + source = query.getRequiredUserId, + relationshipType = sg.RelationshipType.Blocking, + hasRelationship = true, + targets = Some(authorIds)), + sg.SrcRelationship( + source = query.getRequiredUserId, + relationshipType = sg.RelationshipType.BlockedBy, + hasRelationship = true, + targets = Some(authorIds)), + sg.SrcRelationship( + source = query.getRequiredUserId, + relationshipType = sg.RelationshipType.Muting, + hasRelationship = true, + targets = Some(authorIds)) + ), + pageRequest = Some(sg.PageRequest(selectAll = Some(true))), + context = Some(sg.LookupContext(performUnion = Some(true))) + ) + + socialGraph.ids(request).map { result => + val ids = result.ids.toSet + val (removed, kept) = + candidates.partition { candidate => + ids.contains(candidate.features.get(AuthorIdFeature).get) + } + FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/TopKFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/TopKFilter.scala new file mode 100644 index 000000000..f1c1dc628 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/TopKFilter.scala @@ -0,0 +1,31 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * This filter ranks tweets by a score feature and takes top k + */ +case class TopKFilter(scoreFeature: Feature[_, Double], topK: Int) + extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("TopK") + + private val DefaultScore = 0D + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val sorted = candidates + .sortBy(_.features.getOrElse(scoreFeature, DefaultScore))(Ordering[Double].reverse) + val (kept, removed) = sorted.splitAt(topK) + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/TopKOptionalFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/TopKOptionalFilter.scala new file mode 100644 index 000000000..a02f0ecca --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/TopKOptionalFilter.scala @@ -0,0 +1,32 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * This filter ranks tweets by a score feature and takes top k + */ +case class TopKOptionalFilter(scoreFeature: Feature[_, Option[Double]], topK: Int) + extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("TopKOptional") + + private val DefaultScore = 0D + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val sorted = candidates + .sortBy(_.features.getOrElse(scoreFeature, None).getOrElse(DefaultScore))( + Ordering[Double].reverse) + val (kept, removed) = sorted.splitAt(topK) + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/UtegMinFavCountFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/UtegMinFavCountFilter.scala new file mode 100644 index 000000000..6f290849c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/UtegMinFavCountFilter.scala @@ -0,0 +1,32 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.product.scored_tweets.response_transformer.UtegFavListFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object UtegMinFavCountFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("UtegMinFavCount") + + private val UtegMinFavCount = 1 + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (kept, removed) = candidates.partition { candidate => + val nonAuthorFavList = candidate.features + .getOrElse(UtegFavListFeature, Seq.empty) + .filter(!candidate.features.getOrElse(AuthorIdFeature, None).contains(_)) + nonAuthorFavList.size >= UtegMinFavCount + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/UtegTopKFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/UtegTopKFilter.scala new file mode 100644 index 000000000..f1a77c16e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/UtegTopKFilter.scala @@ -0,0 +1,35 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FetchParams +import com.twitter.home_mixer.product.scored_tweets.response_transformer.UtegScoreFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object UtegTopKFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("UtegTopK") + + private val DefaultScore = 0D + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val topK = query.params(FetchParams.UTEGMaxTweetsToFetchParam) + val sorted = candidates.sortBy { candidate => + val utegScore = candidate.features.getOrElse(UtegScoreFeature, DefaultScore) + val ebScore = candidate.features + .getOrElse(EarlybirdScoreFeature, None) + .getOrElse(DefaultScore) + utegScore + ebScore * 2 + }(Ordering[Double].reverse) + val (kept, removed) = sorted.splitAt(topK) + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/AllowLowSignalUserGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/AllowLowSignalUserGate.scala new file mode 100644 index 000000000..acdb1eb80 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/AllowLowSignalUserGate.scala @@ -0,0 +1,26 @@ +package com.twitter.home_mixer.product.scored_tweets.gate + +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableLowSignalUserCheck +import com.twitter.home_mixer.util.SignalUtil +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Continue for low signal users who also follow < 5 users + * + * Check gate param last to only evaluate for eligible users and avoid experimental dilution. + */ +object AllowLowSignalUserGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("AllowLowSignalUser") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + val continue = + SignalUtil.isLowSignalUser(query) && + query.params(EnableLowSignalUserCheck) + + Stitch.value(continue) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/BUILD.bazel index 6d3fd9b1c..e7edb746d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/BUILD.bazel @@ -5,6 +5,7 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/DenyLowSignalUserGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/DenyLowSignalUserGate.scala new file mode 100644 index 000000000..d2df5d2d7 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/DenyLowSignalUserGate.scala @@ -0,0 +1,30 @@ +package com.twitter.home_mixer.product.scored_tweets.gate + +import com.twitter.home_mixer.model.HomeFeatures.SignupSourceFeature +import com.twitter.home_mixer.model.signup.MarchMadness +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableLowSignalUserCheck +import com.twitter.home_mixer.util.SignalUtil +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Continue for all users except low signal users who also follow < 25 users + * + * Check gate param last to only evaluate for eligible users and avoid experimental dilution. + */ +object DenyLowSignalUserGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("DenyLowSignalUser") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + val signupSource = query.features.flatMap(_.getOrElse(SignupSourceFeature, None)) + + val stop = signupSource.contains(MarchMadness) && + SignalUtil.isLowSignalUser(query) && + query.params(EnableLowSignalUserCheck) + + Stitch.value(!stop) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MatchesCountryGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MatchesCountryGate.scala new file mode 100644 index 000000000..f785f1b54 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MatchesCountryGate.scala @@ -0,0 +1,30 @@ +package com.twitter.home_mixer.product.scored_tweets.gate + +import com.twitter.home_mixer.model.HomeFeatures.SignupCountryFeature +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelines.configapi.Param + +/** + * Check whether the signup country code feature or the input country code + * exists within the input country codes param + */ +case class MatchesCountryGate(countryCodes: Param[Seq[String]]) extends Gate[PipelineQuery] { + override val identifier: GateIdentifier = GateIdentifier("MatchesSignupCountry") + + /** + * The main predicate that controls this gate. If this predicate returns true, the gate returns Continue. + */ + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + val countryCode = query.clientContext.countryCode + val signupCountryCode = query.features + .flatMap(_.getOrElse(SignupCountryFeature, None)) + val codeMatchesInput = + Seq(countryCode, signupCountryCode).flatten.exists { code => + query.params(countryCodes).map(_.toLowerCase).contains(code.toLowerCase) + } + Stitch.value(codeMatchesInput) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinCachedTweetsGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinCachedTweetsGate.scala index bdeb33a92..703c75b9a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinCachedTweetsGate.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinCachedTweetsGate.scala @@ -1,6 +1,8 @@ package com.twitter.home_mixer.product.scored_tweets.gate +import com.twitter.home_mixer.model.HomeFeatures.HasRecentFeedbackSinceCacheTtlFeature import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate.identifierSuffix +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableRecentFeedbackCheckParam import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier @@ -25,7 +27,14 @@ case class MinCachedTweetsGate( tweet.candidatePipelineIdentifier.exists( CandidatePipelineIdentifier(_).equals(candidatePipelineIdentifier)) } - Stitch.value(numCachedTweets < minCachedTweets) + + val hasMinCachedTweets = numCachedTweets < minCachedTweets + query.features.map(_.getOrElse(HasRecentFeedbackSinceCacheTtlFeature, false)) match { + case Some(true) => + if (query.params(EnableRecentFeedbackCheckParam)) Stitch.True + else Stitch.value(hasMinCachedTweets) + case _ => Stitch.value(hasMinCachedTweets) + } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/RecentFeedbackCheckGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/RecentFeedbackCheckGate.scala new file mode 100644 index 000000000..aba0484a2 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/RecentFeedbackCheckGate.scala @@ -0,0 +1,27 @@ +package com.twitter.home_mixer.product.scored_tweets.gate + +import com.twitter.home_mixer.model.HomeFeatures.HasRecentFeedbackSinceCacheTtlFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableRecentFeedbackCheckParam +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Continue if a user has no don't like feedback within cached tweets ttl time (3 mins). + * The reason is that if a user clicks don't like, tweets in the cache won't be affected + * and it feels our system has slow response to user's negative feedback + */ + +object RecentFeedbackCheckGate extends Gate[PipelineQuery] { + override val identifier: GateIdentifier = GateIdentifier("RecentFeedbackCheck") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + query.features.map(_.getOrElse(HasRecentFeedbackSinceCacheTtlFeature, false)) match { + case Some(true) => + if (query.params(EnableRecentFeedbackCheckParam)) Stitch.False + else Stitch.True + case _ => Stitch.True + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel index 4347c1093..d42152aec 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel @@ -5,11 +5,13 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala index 796970bec..c4a055e11 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala @@ -1,5 +1,8 @@ package com.twitter.home_mixer.product.scored_tweets.marshaller +import com.twitter.home_mixer.model.HomeFeatures.UserActionsContainsExplicitSignalsFeature +import com.twitter.home_mixer.model.HomeFeatures.UserActionsSizeFeature +import com.twitter.home_mixer.product.scored_tweets.model.QueryMetadata import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller @@ -18,5 +21,13 @@ object ScoredTweetsResponseDomainMarshaller override def apply( query: ScoredTweetsQuery, selections: Seq[CandidateWithDetails] - ): ScoredTweetsResponse = ScoredTweetsResponse(scoredTweets = selections) + ): ScoredTweetsResponse = ScoredTweetsResponse( + scoredTweets = selections, + queryMetadata = Some( + QueryMetadata( + userActionsSize = query.features.get.getOrElse(UserActionsSizeFeature, None), + userActionsContainsExplicitSignals = + Some(query.features.get.getOrElse(UserActionsContainsExplicitSignalsFeature, false)) + )) + ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala index 27486768b..06f59a50d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala @@ -1,8 +1,14 @@ package com.twitter.home_mixer.product.scored_tweets.marshaller import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.model._ +import com.twitter.home_mixer.product.scored_tweets.model.QueryMetadata import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse -import com.twitter.home_mixer.{thriftscala => t} +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityNameFeature +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.TopicContextFunctionalityTypeMarshaller @@ -12,28 +18,80 @@ import com.twitter.product_mixer.core.model.common.identifier.TransportMarshalle * Marshall the domain model into our transport (Thrift) model. */ object ScoredTweetsResponseTransportMarshaller - extends TransportMarshaller[ScoredTweetsResponse, t.ScoredTweetsResponse] { + extends TransportMarshaller[ScoredTweetsResponse, hmt.ScoredTweetsResponse] { override val identifier: TransportMarshallerIdentifier = TransportMarshallerIdentifier("ScoredTweetsResponse") - override def apply(input: ScoredTweetsResponse): t.ScoredTweetsResponse = { + override def apply(input: ScoredTweetsResponse): hmt.ScoredTweetsResponse = { val scoredTweets = input.scoredTweets.map { tweet => - mkScoredTweet(tweet.candidateIdLong, tweet.features) + mkScoredTweet(tweet.candidateIdLong, tweet.features, input.queryMetadata) } - t.ScoredTweetsResponse(scoredTweets) + hmt.ScoredTweetsResponse(scoredTweets) } - private def mkScoredTweet(tweetId: Long, features: FeatureMap): t.ScoredTweet = { + private def mkScoredTweet( + tweetId: Long, + features: FeatureMap, + queryMetadata: Option[QueryMetadata] + ): hmt.ScoredTweet = { val topicFunctionalityType = features .getOrElse(TopicContextFunctionalityTypeFeature, None) .map(TopicContextFunctionalityTypeMarshaller(_)) - t.ScoredTweet( + val predictedScores = hmt.PredictedScores( + favoriteScore = features.getOrElse(PredictedFavoriteScoreFeature, None), + replyScore = features.getOrElse(PredictedReplyScoreFeature, None), + retweetScore = features.getOrElse(PredictedRetweetScoreFeature, None), + replyEngagedByAuthorScore = + features.getOrElse(PredictedReplyEngagedByAuthorScoreFeature, None), + goodClickConvoDescFavoritedOrRepliedScore = features + .getOrElse(PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, None), + goodClickConvoDescUamGt2Score = + features.getOrElse(PredictedGoodClickConvoDescUamGt2ScoreFeature, None), + goodProfileClickScore = features.getOrElse(PredictedGoodProfileClickScoreFeature, None), + videoQualityViewScore = features.getOrElse(PredictedVideoQualityViewScoreFeature, None), + shareScore = features.getOrElse(PredictedShareScoreFeature, None), + dwellScore = features.getOrElse(PredictedDwellScoreFeature, None), + negativeFeedbackV2Score = features.getOrElse(PredictedNegativeFeedbackV2ScoreFeature, None) + ) + + val phoenixPredictedScores = hmt.PredictedScores( + favoriteScore = features.getOrElse(PhoenixPredictedFavoriteScoreFeature, None), + replyScore = features.getOrElse(PhoenixPredictedReplyScoreFeature, None), + retweetScore = features.getOrElse(PhoenixPredictedRetweetScoreFeature, None), + replyEngagedByAuthorScore = None, + goodClickConvoDescFavoritedOrRepliedScore = features + .getOrElse(PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, None), + goodClickConvoDescUamGt2Score = + features.getOrElse(PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature, None), + goodProfileClickScore = + features.getOrElse(PhoenixPredictedGoodProfileClickScoreFeature, None), + videoQualityViewScore = + features.getOrElse(PhoenixPredictedVideoQualityViewScoreFeature, None), + shareScore = features.getOrElse(PhoenixPredictedShareScoreFeature, None), + dwellScore = features.getOrElse(PhoenixPredictedDwellScoreFeature, None), + negativeFeedbackV2Score = + features.getOrElse(PhoenixPredictedNegativeFeedbackV2ScoreFeature, None), + openLinkScore = features.getOrElse(PhoenixPredictedOpenLinkScoreFeature, None), + screenshotScore = features.getOrElse(PhoenixPredictedScreenshotScoreFeature, None), + bookmarkScore = features.getOrElse(PhoenixPredictedBookmarkScoreFeature, None) + ) + + val sourceSignal: Option[hmt.SourceSignal] = features.getOrElse(SourceSignalFeature, None).map { + modelSignal => + hmt.SourceSignal( + id = modelSignal.id, + signalType = modelSignal.signalType, + signalEntity = modelSignal.signalEntity, + authorId = modelSignal.authorId, + ) + } + hmt.ScoredTweet( tweetId = tweetId, authorId = features.get(AuthorIdFeature).get, score = features.get(ScoreFeature), - suggestType = features.get(SuggestTypeFeature), + servedType = features.get(ServedTypeFeature), sourceTweetId = features.getOrElse(SourceTweetIdFeature, None), sourceUserId = features.getOrElse(SourceUserIdFeature, None), quotedTweetId = features.getOrElse(QuotedTweetIdFeature, None), @@ -45,26 +103,48 @@ object ScoredTweetsResponseTransportMarshaller sgsValidLikedByUserIds = Some(features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)), sgsValidFollowedByUserIds = Some(features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)), + validLikedByUserIds = Some(features.getOrElse(ValidLikedByUserIdsFeature, Seq.empty)), topicId = features.getOrElse(TopicIdSocialContextFeature, None), topicFunctionalityType = topicFunctionalityType, ancestors = Some(features.getOrElse(AncestorsFeature, Seq.empty)), isReadFromCache = Some(features.getOrElse(IsReadFromCacheFeature, false)), - streamToKafka = Some(features.getOrElse(StreamToKafkaFeature, false)), exclusiveConversationAuthorId = features.getOrElse(ExclusiveConversationAuthorIdFeature, None), authorMetadata = Some( - t.AuthorMetadata( + hmt.AuthorMetadata( blueVerified = features.getOrElse(AuthorIsBlueVerifiedFeature, false), goldVerified = features.getOrElse(AuthorIsGoldVerifiedFeature, false), grayVerified = features.getOrElse(AuthorIsGrayVerifiedFeature, false), legacyVerified = features.getOrElse(AuthorIsLegacyVerifiedFeature, false), - creator = features.getOrElse(AuthorIsCreatorFeature, false) + creator = features.getOrElse(AuthorIsCreatorFeature, false), + followers = features.getOrElse(AuthorFollowersFeature, None) )), lastScoredTimestampMs = None, candidatePipelineIdentifier = None, tweetUrls = None, - perspectiveFilteredLikedByUserIds = - Some(features.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty)), + perspectiveFilteredLikedByUserIds = None, + predictionRequestId = features.getOrElse(PredictionRequestIdFeature, None), + communityId = features.getOrElse(CommunityIdFeature, None), + communityName = features.getOrElse(CommunityNameFeature, None), + listId = features.getOrElse(ListIdFeature, None), + listName = features.getOrElse(ListNameFeature, None), + isNsfw = features.getOrElse(IsNsfw, None), + visibilityReason = features.getOrElse(VisibilityReason, None), + tweetLanguage = features.getOrElse(TweetLanguageFeature, None), + tweetText = features.getOrElse(TweetTextFeature, None), + tweetTypeMetrics = features.getOrElse(TweetTypeMetricsFeature, None), + debugString = features.getOrElse(DebugStringFeature, None), + hasVideo = Some(features.getOrElse(HasVideoFeature, false)), + videoDurationMs = features.getOrElse(VideoDurationMsFeature, None), + mediaIds = Some(features.getOrElse(TweetMediaIdsFeature, Seq.empty)), + grokAnnotations = features.getOrElse(GrokAnnotationsFeature, None), + predictedScores = Some(predictedScores), + tweetMixerScore = features.getOrElse(TweetMixerScoreFeature, None), + phoenixPredictedScores = Some(phoenixPredictedScores), + sourceSignal = sourceSignal, + userActionsSize = queryMetadata.flatMap(_.userActionsSize), + userActionsContainsExplicitSignals = + queryMetadata.flatMap(_.userActionsContainsExplicitSignals) ) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel index 6414453db..3081d1cf2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel @@ -4,11 +4,8 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsQuery.scala index a2eb3a466..8ff457330 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsQuery.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsQuery.scala @@ -3,7 +3,7 @@ package com.twitter.home_mixer.product.scored_tweets.model import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.model.request.ScoredTweetsProduct +import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.request._ @@ -22,13 +22,17 @@ case class ScoredTweetsQuery( override val features: Option[FeatureMap], override val deviceContext: Option[DeviceContext], override val seenTweetIds: Option[Seq[Long]], - override val qualityFactorStatus: Option[QualityFactorStatus]) + override val qualityFactorStatus: Option[QualityFactorStatus], + override val product: Product, + videoType: Option[t.VideoType] = None, + pinnedRelatedTweetIds: Option[Seq[Long]] = None, + scorePinnedTweetsOnly: Option[Boolean] = None, + immersiveClientMetadata: Option[t.ImmersiveClientMetadata] = None) extends PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext with HasSeenTweetIds with HasQualityFactorStatus { - override val product: Product = ScoredTweetsProduct override def withFeatureMap(features: FeatureMap): ScoredTweetsQuery = copy(features = Some(features)) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala index e9bd7cd61..59c578df5 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala @@ -2,5 +2,16 @@ package com.twitter.home_mixer.product.scored_tweets.model import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling +import com.twitter.product_mixer.core.model.marshalling.HasLength -case class ScoredTweetsResponse(scoredTweets: Seq[CandidateWithDetails]) extends HasMarshalling +case class ScoredTweetsResponse( + scoredTweets: Seq[CandidateWithDetails], + queryMetadata: Option[QueryMetadata] = None) + extends HasMarshalling + with HasLength { + override def length: Int = scoredTweets.length +} + +case class QueryMetadata( + userActionsSize: Option[Int] = None, + userActionsContainsExplicitSignals: Option[Boolean] = None) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel index 065841642..b4ff2a9b8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel @@ -4,7 +4,6 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala index 9720b9344..14241b87d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala @@ -7,32 +7,54 @@ import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.decider.BooleanDeciderParam +import com.twitter.timelines.configapi.decider.DeciderBoundedParam import com.twitter.util.Duration object ScoredTweetsParam { val SupportedClientFSName = "scored_tweets_supported_client" - object CandidatePipeline { - object EnableInNetworkParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsInNetworkCandidatePipeline) + object CandidateSourceParams { + object EnableCommunitiesCandidateSourceParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_earlybird_communities_candidate_source", + default = false + ) - object EnableTweetMixerParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsTweetMixerCandidatePipeline) + object EnableInNetworkCandidateSourceParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_in_network_candidate_source", + default = true + ) - object EnableUtegParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsUtegCandidatePipeline) + object EnableStaticSourceParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_static_source", + default = false + ) - object EnableFrsParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsFrsCandidatePipeline) + object EnableUTEGCandidateSourceParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_uteg_candidate_source", + default = true + ) - object EnableListsParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsListsCandidatePipeline) + object InNetworkIncludeRepliesParam + extends FSParam[Boolean]( + name = "scored_tweets_in_network_include_replies", + default = true + ) - object EnablePopularVideosParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsPopularVideosCandidatePipeline) + object InNetworkIncludeRetweetsParam + extends FSParam[Boolean]( + name = "scored_tweets_in_network_include_retweets", + default = true + ) - object EnableBackfillParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsBackfillCandidatePipeline) + object InNetworkIncludeExtendedRepliesParam + extends FSParam[Boolean]( + name = "scored_tweets_in_network_include_extended_replies", + default = true + ) } object EnableBackfillCandidatePipelineParam @@ -41,27 +63,144 @@ object ScoredTweetsParam { default = true ) - object QualityFactor { - object InNetworkMaxTweetsToScoreParam + object EnableContentExplorationCandidatePipelineParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_content_exploration_candidate_pipeline", + default = false + ) + + object ContentExplorationCandidateVersionParam + extends FSParam[String]( + name = "scored_tweets_enable_content_exploration_candidate_version", + default = "v1_" + ) + + object EnableContentExplorationScoreScribingParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_content_exploration_score_scribing", + default = false + ) + + object EnableContentExplorationCandidateMaxCountParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_content_exploration_candidate_max_count", + default = false + ) + + object EnableContentExplorationSimclusterColdPostsCandidateBoostingParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_content_exploration_simcluster_cold_posts_candidate_boosting", + default = false + ) + + object ContentExplorationBoostPosParam + extends FSBoundedParam[Int]( + name = "scored_tweets_content_exploration_boost_pos", + default = 100, + min = 0, + max = 1000 + ) + + object EnableDeepRetrievalMixedCandidateBoostingParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_deep_retrieval_mixed_candidate_boosting", + default = false + ) + + object CategoryColdStartTierOneProbabilityParam + extends FSBoundedParam[Double]( + name = "scored_tweets_category_cold_start_tier_one_probability", + default = 0, + min = 0, + max = 1 + ) + + object CategoryColdStartProbabilisticReturnParam + extends FSBoundedParam[Double]( + name = "scored_tweets_category_cold_start_probabilistic_return", + default = 0, + min = 0, + max = 1 + ) + + object ContentExplorationViewerMaxFollowersParam + extends FSBoundedParam[Int]( + name = "scored_tweets_content_exploration_viewer_max_followers", + default = 100000, + min = 0, + max = 1000000000 + ) + + object EnableContentExplorationMixedCandidateBoostingParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_content_exploration_mixed_candidate_boosting", + default = false + ) + + object DeepRetrievalBoostPosParam + extends FSBoundedParam[Int]( + name = "scored_tweets_deep_retrieval_boost_pos", + default = 100, + min = 0, + max = 1000 + ) + + object DeepRetrievalI2iProbabilityParam + extends FSBoundedParam[Double]( + name = "scored_tweets_deep_retrieval_i2i_probability", + default = 0, + min = 0, + max = 1 + ) + + object FetchParams { + object FRSMaxTweetsToFetchParam extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_earlybird_max_tweets_to_score", - default = 500, + name = "scored_tweets_frs_max_tweets_to_fetch", + default = 100, min = 0, max = 10000 ) - object UtegMaxTweetsToScoreParam + object InNetworkMaxTweetsToFetchParam extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_uteg_max_tweets_to_score", - default = 500, + name = "scored_tweets_in_network_max_tweets_to_fetch", + default = 600, min = 0, max = 10000 ) - object FrsMaxTweetsToScoreParam + object TweetMixerMaxTweetsToFetchParam extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_frs_max_tweets_to_score", - default = 500, + name = "scored_tweets_tweet_mixer_max_tweets_to_fetch", + default = 400, + min = 0, + max = 10000 + ) + + object UTEGMaxTweetsToFetchParam + extends FSBoundedParam[Int]( + name = "scored_tweets_uteg_max_tweets_to_fetch", + default = 300, + min = 0, + max = 10000 + ) + } + + object QualityFactor { + + object InNetworkMaxTweetsToScoreParam + extends FSBoundedParam[Int]( + name = "scored_tweets_quality_factor_earlybird_max_tweets_to_score", + default = 600, + min = 0, + max = 10000 + ) + + object UtegMaxTweetsToScoreParam + extends FSBoundedParam[Int]( + name = "scored_tweets_quality_factor_uteg_max_tweets_to_score", + default = 300, min = 0, max = 10000 ) @@ -69,7 +208,7 @@ object ScoredTweetsParam { object TweetMixerMaxTweetsToScoreParam extends FSBoundedParam[Int]( name = "scored_tweets_quality_factor_tweet_mixer_max_tweets_to_score", - default = 500, + default = 400, min = 0, max = 10000 ) @@ -77,23 +216,23 @@ object ScoredTweetsParam { object ListsMaxTweetsToScoreParam extends FSBoundedParam[Int]( name = "scored_tweets_quality_factor_lists_max_tweets_to_score", - default = 500, + default = 100, min = 0, max = 100 ) - object PopularVideosMaxTweetsToScoreParam + object BackfillMaxTweetsToScoreParam extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_popular_videos_max_tweets_to_score", - default = 40, + name = "scored_tweets_quality_factor_backfill_max_tweets_to_score", + default = 200, min = 0, max = 10000 ) - object BackfillMaxTweetsToScoreParam + object CommunitiesMaxTweetsToScoreParam extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_backfill_max_tweets_to_score", - default = 500, + name = "scored_tweets_quality_factor_communities_max_tweets_to_score", + default = 100, min = 0, max = 10000 ) @@ -102,25 +241,17 @@ object ScoredTweetsParam { object ServerMaxResultsParam extends FSBoundedParam[Int]( name = "scored_tweets_server_max_results", - default = 120, + default = 50, min = 1, - max = 500 + max = 1500 ) - object MaxInNetworkResultsParam + object DefaultRequestedMaxResultsParam extends FSBoundedParam[Int]( - name = "scored_tweets_max_in_network_results", - default = 60, + name = "scored_tweets_default_requested_max_results", + default = 50, min = 1, - max = 500 - ) - - object MaxOutOfNetworkResultsParam - extends FSBoundedParam[Int]( - name = "scored_tweets_max_out_of_network_results", - default = 60, - min = 1, - max = 500 + max = 1500 ) object CachedScoredTweets { @@ -144,179 +275,179 @@ object ScoredTweetsParam { ) } - object Scoring { - object HomeModelParam - extends FSParam[String](name = "scored_tweets_home_model", default = "Home") - - object ModelWeights { - - object FavParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_fav", - default = 1.0, - min = 0.0, - max = 100.0 - ) - - object RetweetParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_retweet", - default = 1.0, - min = 0.0, - max = 100.0 - ) - - object ReplyParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_reply", - default = 1.0, - min = 0.0, - max = 100.0 - ) - - object GoodProfileClickParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_good_profile_click", - default = 1.0, - min = 0.0, - max = 1000000.0 - ) - - object VideoPlayback50Param - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_video_playback50", - default = 1.0, - min = 0.0, - max = 100.0 - ) - - object ReplyEngagedByAuthorParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_reply_engaged_by_author", - default = 1.0, - min = 0.0, - max = 200.0 - ) - - object GoodClickParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_good_click", - default = 1.0, - min = 0.0, - max = 1000000.0 - ) - - object GoodClickV2Param - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_good_click_v2", - default = 1.0, - min = 0.0, - max = 1000000.0 - ) - - object TweetDetailDwellParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_tweet_detail_dwell", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object ProfileDwelledParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_profile_dwelled", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object BookmarkParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_bookmark", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object ShareParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_share", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object ShareMenuClickParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_share_menu_click", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object NegativeFeedbackV2Param - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_negative_feedback_v2", - default = 1.0, - min = -1000.0, - max = 0.0 - ) - - object ReportParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_report", - default = 1.0, - min = -20000.0, - max = 0.0 - ) - - object WeakNegativeFeedbackParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_weak_negative_feedback", - default = 0.0, - min = -1000.0, - max = 0.0 - ) - - object StrongNegativeFeedbackParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_strong_negative_feedback", - default = 0.0, - min = -1000.0, - max = 0.0 - ) - } - } + object FeatureHydration { - object EnableSimClustersSimilarityFeatureHydrationDeciderParam - extends BooleanDeciderParam(decider = DeciderKey.EnableSimClustersSimilarityFeatureHydration) + object EnableRealTimeEntityRealGraphFeaturesParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_real_time_entity_real_graph_features", + default = false + ) + + object EnableFollowedUserScoreBackfillFeaturesParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_followed_user_score_backfill_features", + default = false + ) - object CompetitorSetParam - extends FSParam[Set[Long]](name = "scored_tweets_competitor_list", default = Set.empty) + object EnableSgsMutuallyFollowedUserFeaturesParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_sgs_mutually_followed_user_features", + default = false + ) + + object EnableTopicSocialProofFeaturesParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_topic_social_proof_features", + default = false + ) + + object EnableMediaClusterFeatureHydrationParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_media_cluster_feature", + default = false + ) + + object EnableMediaCompletionRateFeatureHydrationParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_media_completion_rate_feature", + default = false + ) + + object EnableImageClusterFeatureHydrationParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_image_cluster_feature", + default = false + ) + + object EnableClipImagesClusterIdFeatureHydrationParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_clip_images_cluster_id_feature", + default = false + ) + + object EnableMultiModalEmbeddingsFeatureHydratorParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_multi_modal_embeddings_feature_hydrator", + default = false + ) + + object EnableTweetTextV8EmbeddingFeatureParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_tweet_text_v8_embedding_feature", + default = false + ) - object CompetitorURLSeqParam - extends FSParam[Seq[String]](name = "scored_tweets_competitor_url_list", default = Seq.empty) + object EnableUserEngagedLanguagesFeaturesParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_user_engaged_languages_features", + default = false + ) + + object EnableUserIdentifierFeaturesParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_user_identifier_features", + default = false + ) + + object EnableUserHistoryEventsFeaturesParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_user_history_events_features", + default = false + ) + + object EnableUserActionsFeatureParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_user_actions_feature", + default = false + ) + + object EnableDenseUserActionsHydrationParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_dense_user_actions_feature", + default = false + ) + + object EnableMediaClusterDecayParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_media_cluster_decay", + default = false + ) + + object EnableImageClusterDecayParam + extends FSParam[Boolean]( + name = "scored_tweets_feature_hydration_enable_image_cluster_decay", + default = false + ) + + object UserHistoryEventsLengthParam + extends FSBoundedParam[Int]( + name = "scored_tweets_feature_hydration_user_history_events_length", + default = 50, + min = 0, + max = 1000 + ) + + object TwhinDiversityRescoringWeightParam + extends FSBoundedParam[Double]( + name = "scored_tweets_feature_hydration_twhin_diversity_rescoring_weight", + default = 0.0, + min = -100.0, + max = 100.0 + ) - object BlueVerifiedAuthorInNetworkMultiplierParam + object TwhinDiversityRescoringRatioParam + extends FSBoundedParam[Double]( + name = "scored_tweets_feature_hydration_twhin_diversity_rescoring_ratio", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object CategoryDiversityRescoringWeightParam + extends FSBoundedParam[Double]( + name = "scored_tweets_feature_hydration_category_diversity_rescoring_weight", + default = 0.0, + min = -1.0, + max = 1.0 + ) + + object CategoryDiversityKParam + extends FSBoundedParam[Int]( + name = "scored_tweets_feature_hydration_category_diversity_k", + default = 5, + min = 1, + max = 100 + ) + } + + object ControlAiShowLessScaleFactorParam extends FSBoundedParam[Double]( - name = "scored_tweets_blue_verified_author_in_network_multiplier", - default = 4.0, + name = "scored_tweets_control_ai_show_less_scale_factor", + default = 0.05, min = 0.0, - max = 100.0 + max = 1.0 ) - object BlueVerifiedAuthorOutOfNetworkMultiplierParam + object ControlAiShowMoreScaleFactorParam extends FSBoundedParam[Double]( - name = "scored_tweets_blue_verified_author_out_of_network_multiplier", - default = 2.0, + name = "scored_tweets_control_ai_show_more_scale_factor", + default = 20.0, min = 0.0, - max = 100.0 + max = 1000.0 + ) + + object ControlAiEmbeddingSimilarityThresholdParam + extends FSBoundedParam[Double]( + name = "scored_tweets_control_ai_embedding_similarity_threshold", + default = 0.67, + min = 0.0, + max = 1.0 ) object CreatorInNetworkMultiplierParam extends FSBoundedParam[Double]( name = "scored_tweets_creator_in_network_multiplier", - default = 1.1, + default = 1.0, min = 0.0, max = 100.0 ) @@ -324,7 +455,7 @@ object ScoredTweetsParam { object CreatorOutOfNetworkMultiplierParam extends FSBoundedParam[Double]( name = "scored_tweets_creator_out_of_network_multiplier", - default = 1.3, + default = 1.0, min = 0.0, max = 100.0 ) @@ -332,30 +463,419 @@ object ScoredTweetsParam { object OutOfNetworkScaleFactorParam extends FSBoundedParam[Double]( name = "scored_tweets_out_of_network_scale_factor", - default = 1.0, + default = 0.75, + min = 0.0, + max = 100.0 + ) + + object ReplyScaleFactorParam + extends FSBoundedParam[Double]( + name = "scored_tweets_reply_scale_factor", + default = 0.75, min = 0.0, max = 100.0 ) + object EnableMediaDedupingParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_media_deduping", + default = false + ) + + object EnableMediaClusterDedupingParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_media_cluster_deduping", + default = false + ) + + object EnableClipImageClusterDedupingParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_clip_image_cluster_deduping", + default = false + ) + object EnableScribeScoredCandidatesParam - extends FSParam[Boolean](name = "scored_tweets_enable_scribing", default = false) + extends FSParam[Boolean]( + name = "scored_tweets_enable_scribing", + default = false + ) - object EarlybirdTensorflowModel { + object EnableCacheRetrievalSignalParam + extends FSParam[Boolean]( + name = "scored_tweets_cache_retrieval_signal", + default = false + ) + + object EnableCacheRequestInfoParam + extends FSParam[Boolean]( + name = "scored_tweets_cache_request_info_signal", + default = false + ) + object EnableScoredPhoenixCandidatesKafkaSideEffectParam + extends FSParam[Boolean]( + name = "scored_tweets_scored_phoenix_candidates_kafka_side_effect", + default = false + ) + + object LiveContentScaleFactorParam + extends DeciderBoundedParam[Double]( + DeciderKey.LiveSpacesFactor, + default = 1.0, + min = 0.1, + max = 10000.0 + ) + + object EarlybirdTensorflowModel { object InNetworkParam extends FSParam[String]( name = "scored_tweets_in_network_earlybird_tensorflow_model", - default = "timelines_recap_replica") + default = "timelines_recap_replica" + ) object FrsParam extends FSParam[String]( name = "scored_tweets_frs_earlybird_tensorflow_model", - default = "timelines_rectweet_replica") + default = "timelines_rectweet_replica" + ) object UtegParam extends FSParam[String]( name = "scored_tweets_uteg_earlybird_tensorflow_model", - default = "timelines_rectweet_replica") + default = "timelines_rectweet_replica" + ) } + object MtlNormalization { + + object EnableMtlNormalizationParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_mtl_normalization", + default = true + ) + + object AlphaParam + extends DeciderBoundedParam[Double]( + decider = DeciderKey.MtlNormalizationAlpha, + default = 100.0, + min = 0.0, + max = 100.0 + ) + + object BetaParam + extends FSBoundedParam[Long]( + name = "scored_tweets_mtl_normalization_beta", + default = 100000000L, + min = 0L, + max = 1000000000L + ) + + object GammaParam + extends FSBoundedParam[Long]( + name = "scored_tweets_mtl_normalization_gamma", + default = 5000000L, + min = 1L, + max = 100000000L + ) + } + + object EarlybirdMaxResultsPerPartitionParam + extends FSBoundedParam[Int]( + name = "scored_tweets_earlybird_max_results_per_partition", + default = 300, + min = 0, + max = 1000 + ) + + object TweetMixerRankingModeForStatsRecallAtKParam + extends FSParam[String]( + name = "scored_tweets_tweet_mixer_ranking_mode_for_stats_recall_at_k", + default = "Interleave" + ) + + object EnablePublishCommonFeaturesKafkaDeciderParam + extends BooleanDeciderParam(decider = DeciderKey.EnablePublishCommonFeaturesKafka) + + object AuthorDiversityDecayFactor + extends FSBoundedParam[Double]( + name = "scored_tweets_author_diversity_decay_factor", + default = 0.5, + min = 0.0, + max = 1.0, + ) + + object AuthorDiversityOutNetworkDecayFactor + extends FSBoundedParam[Double]( + name = "scored_tweets_author_diversity_out_network_decay_factor", + default = 0.5, + min = 0.0, + max = 1.0, + ) + object AuthorDiversityInNetworkDecayFactor + extends FSBoundedParam[Double]( + name = "scored_tweets_author_diversity_in_network_decay_factor", + default = 0.5, + min = 0.0, + max = 1.0, + ) + + object AuthorDiversityFloor + extends FSBoundedParam[Double]( + name = "scored_tweets_author_diversity_floor", + default = 0.25, + min = 0.0, + max = 1.0, + ) + + object AuthorDiversityOutNetworkFloor + extends FSBoundedParam[Double]( + name = "scored_tweets_author_diversity_out_network_floor", + default = 0.25, + min = 0.0, + max = 1.0, + ) + object AuthorDiversityInNetworkFloor + extends FSBoundedParam[Double]( + name = "scored_tweets_author_diversity_in_network_floor", + default = 0.25, + min = 0.0, + max = 1.0, + ) + + object SmallFollowGraphAuthorDiversityDecayFactor + extends FSBoundedParam[Double]( + name = "scored_tweets_small_follow_graph_author_diversity_decay_factor", + default = 0.5, + min = 0.0, + max = 1.0, + ) + + object SmallFollowGraphAuthorDiversityFloor + extends FSBoundedParam[Double]( + name = "scored_tweets_small_follow_graph_author_diversity_floor", + default = 0.25, + min = 0.0, + max = 1.0, + ) + + object EnableDeepRetrievalMaxCountParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_deep_retrieval_max_count", + default = false + ) + + object DeepRetrievalMaxCountParam + extends FSBoundedParam[Int]( + name = "scored_tweets_deep_retrieval_max_count", + default = 1, + min = 0, + max = 1000 + ) + + object EnableEvergreenDeepRetrievalMaxCountParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_evergreen_deep_retrieval_max_count", + default = false + ) + + object EvergreenDeepRetrievalMaxCountParam + extends FSBoundedParam[Int]( + name = "scored_tweets_evergreen_deep_retrieval_max_count", + default = 1, + min = 0, + max = 1000 + ) + + object EnableEvergreenDeepRetrievalCrossBorderMaxCountParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_evergreen_deep_retrieval_cross_border_max_count", + default = false + ) + + object EvergreenDeepRetrievalCrossBorderMaxCountParam + extends FSBoundedParam[Int]( + name = "scored_tweets_evergreen_deep_retrieval_cross_border_max_count", + default = 1, + min = 0, + max = 1000 + ) + object EnableControlAiParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_control_ai", + default = false + ) + + object EnableHeartbeatOptimizerWeightsParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_heartbeat_optimizer_weights", + default = false + ) + + object HeartbeatOptimizerParamsMHPkey + extends FSParam[String]( + name = "scored_tweets_heartbeat_optimizer_params_mh_pkey", + default = "0" + ) + + object EnableHeuristicScoringPipeline + extends FSParam[Boolean]( + name = "scored_tweets_enable_heuristic_scoring_pipeline", + default = true + ) + + object EnablePhoenixScoreParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_phoenix_score", + default = false + ) + + object EnablePhoenixRescoreParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_phoenix_rescore", + default = false + ) + + object EnableColdStartFilterParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_cold_start_filter", + default = false + ) + + object EnableImpressionBasedAuthorDecay + extends FSParam[Boolean]( + name = "scored_tweets_enable_impression_based_author_decay", + default = false + ) + + object EnableCandidateSourceDiversityDecay + extends FSParam[Boolean]( + name = "scored_tweets_enable_candidate_source_diversity_decay", + default = false + ) + + object CandidateSourceDiversityDecayFactor + extends FSBoundedParam[Double]( + name = "scored_tweets_candidate_source_diversity_decay_factor", + default = 0.9, + min = 0.0, + max = 1.0, + ) + + object CandidateSourceDiversityFloor + extends FSBoundedParam[Double]( + name = "scored_tweets_candidate_source_diversity_floor", + default = 0.8, + min = 0.0, + max = 1.0, + ) + + object EnableHomeMixerFeaturesService + extends FSParam[Boolean]( + name = "scored_tweets_enable_home_mixer_features_service", + default = false + ) + + object GrokSlopScoreDecayValueParam + extends FSBoundedParam[Double]( + name = "scored_tweets_grok_slop_score_decay_value", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + object MultiModalEmbeddingRescorerGammaParam + extends FSBoundedParam[Double]( + name = "scored_tweets_multi_modal_embedding_rescorer_gamma", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object MultiModalEmbeddingRescorerMinScoreParam + extends FSBoundedParam[Double]( + name = "scored_tweets_multi_modal_embedding_rescorer_min_score", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + object EnableContentFeatureFromTesService + extends FSParam[Boolean]( + name = "scored_tweets_enable_home_mixer_feature_tweet_entity_service", + default = false + ) + + object EnableLowSignalUserCheck + extends FSParam[Boolean]( + name = "scored_tweets_enable_low_signal_user_check", + default = false + ) + + object LowSignalUserMaxSignalCount + extends FSBoundedParam[Int]( + name = "scored_tweets_low_signal_user_max_signal_count", + default = 10, + min = 0, + max = 100000 + ) + + object MultiModalEmbeddingRescorerNumCandidatesParam + extends FSBoundedParam[Int]( + name = "scored_tweets_multi_modal_embedding_rescorer_num_candidates", + default = 70, + min = 1, + max = 500 + ) + + object EnableScoredCandidateFeatureKeysKafkaPublishingParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_scored_candidate_feature_keys_kafka_publishing", + default = false + ) + + object EnableEarlybirdCommunitiesQueryLinearRankingParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_earlybird_communities_query_linear_ranking", + default = false + ) + + object EarlyBirdCommunitiesMaxSearchResultsParam + extends FSBoundedParam[Int]( + name = "scored_tweets_earlybird_communities_max_search_results", + default = 100, + min = 0, + max = 1000 + ) + + object EnableRecentFeedbackCheckParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_recent_feedback_check", + default = false + ) + + object ScribedScoredCandidateNumParam + extends FSBoundedParam[Int]( + name = "scored_tweets_scribed_scored_candidate_num", + default = 2, + min = 0, + max = 2000 + ) + + object EnableRecentEngagementCacheRefreshParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_recent_engagement_cache_refresh", + default = false + ) + + object EnableLanguageFilter + extends FSParam[Boolean]( + name = "scored_tweets_enable_language_filter", + default = false + ) + + object EnableGrokAutoTranslateLanguageFilter + extends FSParam[Boolean]( + name = "scored_tweets_enable_grok_auto_translate_language_filter", + default = false + ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala index 10b9de49d..6c3239859 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala @@ -4,6 +4,8 @@ import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam._ import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.servo.decider.DeciderKeyName +import com.twitter.timelines.configapi.FSName +import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @@ -12,78 +14,148 @@ class ScoredTweetsParamConfig @Inject() () extends ProductParamConfig { override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableScoredTweetsProduct override val supportedClientFSName: String = SupportedClientFSName - override val booleanDeciderOverrides = Seq( - CandidatePipeline.EnableBackfillParam, - CandidatePipeline.EnableTweetMixerParam, - CandidatePipeline.EnableFrsParam, - CandidatePipeline.EnableInNetworkParam, - CandidatePipeline.EnableListsParam, - CandidatePipeline.EnablePopularVideosParam, - CandidatePipeline.EnableUtegParam, - ScoredTweetsParam.EnableSimClustersSimilarityFeatureHydrationDeciderParam + override val booleanDeciderOverrides = Seq(EnablePublishCommonFeaturesKafkaDeciderParam) + + override val boundedDoubleDeciderOverrides = Seq( + LiveContentScaleFactorParam, + MtlNormalization.AlphaParam ) override val booleanFSOverrides = Seq( + CandidateSourceParams.EnableCommunitiesCandidateSourceParam, + CandidateSourceParams.EnableInNetworkCandidateSourceParam, + CandidateSourceParams.InNetworkIncludeRepliesParam, + CandidateSourceParams.InNetworkIncludeRetweetsParam, + CandidateSourceParams.InNetworkIncludeExtendedRepliesParam, + CandidateSourceParams.EnableUTEGCandidateSourceParam, + CandidateSourceParams.EnableStaticSourceParam, EnableBackfillCandidatePipelineParam, - EnableScribeScoredCandidatesParam + EnableContentExplorationCandidatePipelineParam, + EnableContentExplorationSimclusterColdPostsCandidateBoostingParam, + EnableContentExplorationCandidateMaxCountParam, + EnableContentExplorationScoreScribingParam, + EnableContentExplorationMixedCandidateBoostingParam, + EnableDeepRetrievalMixedCandidateBoostingParam, + EnableScribeScoredCandidatesParam, + EnableCacheRetrievalSignalParam, + EnableCacheRequestInfoParam, + EnableScoredPhoenixCandidatesKafkaSideEffectParam, + FeatureHydration.EnableFollowedUserScoreBackfillFeaturesParam, + FeatureHydration.EnableRealTimeEntityRealGraphFeaturesParam, + FeatureHydration.EnableTopicSocialProofFeaturesParam, + FeatureHydration.EnableTweetTextV8EmbeddingFeatureParam, + FeatureHydration.EnableMediaClusterFeatureHydrationParam, + FeatureHydration.EnableMediaCompletionRateFeatureHydrationParam, + FeatureHydration.EnableClipImagesClusterIdFeatureHydrationParam, + FeatureHydration.EnableMultiModalEmbeddingsFeatureHydratorParam, + FeatureHydration.EnableUserEngagedLanguagesFeaturesParam, + FeatureHydration.EnableUserIdentifierFeaturesParam, + FeatureHydration.EnableUserHistoryEventsFeaturesParam, + FeatureHydration.EnableUserActionsFeatureParam, + FeatureHydration.EnableDenseUserActionsHydrationParam, + FeatureHydration.EnableMediaClusterDecayParam, + FeatureHydration.EnableImageClusterDecayParam, + EnableControlAiParam, + EnableHeartbeatOptimizerWeightsParam, + EnableHeuristicScoringPipeline, + EnablePhoenixScoreParam, + EnablePhoenixRescoreParam, + EnableColdStartFilterParam, + EnableImpressionBasedAuthorDecay, + EnableCandidateSourceDiversityDecay, + EnableHomeMixerFeaturesService, + EnableContentFeatureFromTesService, + EnableLowSignalUserCheck, + EnableDeepRetrievalMaxCountParam, + EnableEvergreenDeepRetrievalMaxCountParam, + EnableEvergreenDeepRetrievalCrossBorderMaxCountParam, + EnableScoredCandidateFeatureKeysKafkaPublishingParam, + EnableEarlybirdCommunitiesQueryLinearRankingParam, + EnableRecentFeedbackCheckParam, + EnableMediaDedupingParam, + EnableMediaClusterDedupingParam, + EnableClipImageClusterDedupingParam, + MtlNormalization.EnableMtlNormalizationParam, + EnableRecentEngagementCacheRefreshParam, + EnableLanguageFilter, + EnableGrokAutoTranslateLanguageFilter ) override val boundedIntFSOverrides = Seq( CachedScoredTweets.MinCachedTweetsParam, - MaxInNetworkResultsParam, - MaxOutOfNetworkResultsParam, + EarlybirdMaxResultsPerPartitionParam, + ServerMaxResultsParam, + DefaultRequestedMaxResultsParam, + ContentExplorationBoostPosParam, + ContentExplorationViewerMaxFollowersParam, + DeepRetrievalBoostPosParam, + FeatureHydration.UserHistoryEventsLengthParam, + FetchParams.FRSMaxTweetsToFetchParam, + FetchParams.InNetworkMaxTweetsToFetchParam, + FetchParams.TweetMixerMaxTweetsToFetchParam, + FetchParams.UTEGMaxTweetsToFetchParam, QualityFactor.BackfillMaxTweetsToScoreParam, QualityFactor.TweetMixerMaxTweetsToScoreParam, - QualityFactor.FrsMaxTweetsToScoreParam, QualityFactor.InNetworkMaxTweetsToScoreParam, QualityFactor.ListsMaxTweetsToScoreParam, - QualityFactor.PopularVideosMaxTweetsToScoreParam, QualityFactor.UtegMaxTweetsToScoreParam, - ServerMaxResultsParam + QualityFactor.CommunitiesMaxTweetsToScoreParam, + DeepRetrievalMaxCountParam, + EvergreenDeepRetrievalMaxCountParam, + EvergreenDeepRetrievalCrossBorderMaxCountParam, + ScribedScoredCandidateNumParam, + LowSignalUserMaxSignalCount, + EarlyBirdCommunitiesMaxSearchResultsParam, + FeatureHydration.CategoryDiversityKParam, + MultiModalEmbeddingRescorerNumCandidatesParam + ) + + override val boundedLongFSOverrides = Seq( + MtlNormalization.BetaParam, + MtlNormalization.GammaParam, ) override val boundedDurationFSOverrides = Seq( - CachedScoredTweets.TTLParam + CachedScoredTweets.TTLParam, ) override val stringFSOverrides = Seq( - Scoring.HomeModelParam, + ContentExplorationCandidateVersionParam, EarlybirdTensorflowModel.InNetworkParam, EarlybirdTensorflowModel.FrsParam, - EarlybirdTensorflowModel.UtegParam + EarlybirdTensorflowModel.UtegParam, + HeartbeatOptimizerParamsMHPkey, + TweetMixerRankingModeForStatsRecallAtKParam, ) override val boundedDoubleFSOverrides = Seq( - BlueVerifiedAuthorInNetworkMultiplierParam, - BlueVerifiedAuthorOutOfNetworkMultiplierParam, + CategoryColdStartTierOneProbabilityParam, + CategoryColdStartProbabilisticReturnParam, + ControlAiShowLessScaleFactorParam, + ControlAiShowMoreScaleFactorParam, + ControlAiEmbeddingSimilarityThresholdParam, CreatorInNetworkMultiplierParam, CreatorOutOfNetworkMultiplierParam, + DeepRetrievalI2iProbabilityParam, OutOfNetworkScaleFactorParam, - // Model Weights - Scoring.ModelWeights.FavParam, - Scoring.ModelWeights.ReplyParam, - Scoring.ModelWeights.RetweetParam, - Scoring.ModelWeights.GoodClickParam, - Scoring.ModelWeights.GoodClickV2Param, - Scoring.ModelWeights.GoodProfileClickParam, - Scoring.ModelWeights.ReplyEngagedByAuthorParam, - Scoring.ModelWeights.VideoPlayback50Param, - Scoring.ModelWeights.ReportParam, - Scoring.ModelWeights.NegativeFeedbackV2Param, - Scoring.ModelWeights.TweetDetailDwellParam, - Scoring.ModelWeights.ProfileDwelledParam, - Scoring.ModelWeights.BookmarkParam, - Scoring.ModelWeights.ShareParam, - Scoring.ModelWeights.ShareMenuClickParam, - Scoring.ModelWeights.StrongNegativeFeedbackParam, - Scoring.ModelWeights.WeakNegativeFeedbackParam + ReplyScaleFactorParam, + AuthorDiversityDecayFactor, + AuthorDiversityOutNetworkDecayFactor, + AuthorDiversityInNetworkDecayFactor, + SmallFollowGraphAuthorDiversityDecayFactor, + AuthorDiversityFloor, + AuthorDiversityOutNetworkFloor, + AuthorDiversityInNetworkFloor, + CandidateSourceDiversityDecayFactor, + CandidateSourceDiversityFloor, + SmallFollowGraphAuthorDiversityFloor, + FeatureHydration.TwhinDiversityRescoringWeightParam, + FeatureHydration.TwhinDiversityRescoringRatioParam, + FeatureHydration.CategoryDiversityRescoringWeightParam, + GrokSlopScoreDecayValueParam, + MultiModalEmbeddingRescorerGammaParam, + MultiModalEmbeddingRescorerMinScoreParam ) - override val longSetFSOverrides = Seq( - CompetitorSetParam - ) - - override val stringSeqFSOverrides = Seq( - CompetitorURLSeqParam - ) + override def longSetFSOverrides: Seq[Param[Set[Long]] with FSName] = Seq.empty } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel index 15fc94f47..47080073d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel @@ -3,17 +3,16 @@ scala_library( compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "src/thrift/com/twitter/timelineranker:thrift-scala", "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelines:util", "timelines/src/main/scala/com/twitter/timelines/common/model", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/ContentExplorationQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/ContentExplorationQueryTransformer.scala new file mode 100644 index 000000000..98c28189f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/ContentExplorationQueryTransformer.scala @@ -0,0 +1,26 @@ +package com.twitter.home_mixer.product.scored_tweets.query_transformer + +import com.twitter.home_mixer.functional_component.feature_hydrator.UserSubLevelCategoriesFeature +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ContentExplorationCandidateVersionParam +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer + +case class ContentExplorationQueryRequest( + userCategories: Seq[(String, Double)], + version: String) + +object ContentExplorationQueryTransformer + extends CandidatePipelineQueryTransformer[ + ScoredTweetsQuery, + ContentExplorationQueryRequest + ] { + + override def transform(query: ScoredTweetsQuery): ContentExplorationQueryRequest = { + ContentExplorationQueryRequest( + userCategories = query.features.get + .getOrElse(UserSubLevelCategoriesFeature, Seq.empty[(Long, Double)]) + .map { case (id, score) => (id.toString, score) }, + version = query.params(ContentExplorationCandidateVersionParam) + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala index 2592ec82e..1c52b1a0c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala @@ -2,10 +2,11 @@ package com.twitter.home_mixer.product.scored_tweets.query_transformer import com.twitter.conversions.DurationOps._ import com.twitter.core_workflows.user_model.{thriftscala => um} +import com.twitter.home_mixer.functional_component.feature_hydrator.FrsSeedUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUserIdsFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FetchParams import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerFrsQueryTransformer._ import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier @@ -18,8 +19,6 @@ import com.twitter.timelines.model.candidate.CandidateTweetSourceId object TimelineRankerFrsQueryTransformer { private val DefaultSinceDuration = 24.hours private val ExpandedSinceDuration = 48.hours - private val MaxTweetsToFetch = 100 - private val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOption(includeOriginalTweetsAndQuotes = true) @@ -36,13 +35,14 @@ object TimelineRankerFrsQueryTransformer { case class TimelineRankerFrsQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( - override val candidatePipelineIdentifier: CandidatePipelineIdentifier, - override val maxTweetsToFetch: Int = MaxTweetsToFetch) + override val candidatePipelineIdentifier: CandidatePipelineIdentifier) extends CandidatePipelineQueryTransformer[Query, t.RecapQuery] with TimelineRankerQueryTransformer[Query] { override val candidateTweetSourceId = CandidateTweetSourceId.FrsTweet override val options = tweetKindOptions + override def maxTweetsToFetch(query: Query): Int = + query.params(FetchParams.FRSMaxTweetsToFetchParam) override def getTensorflowModel(query: Query): Option[String] = { Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.FrsParam)) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala index 4514dc2c4..905e22db4 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala @@ -5,6 +5,7 @@ import com.twitter.core_workflows.user_model.{thriftscala => um} import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FetchParams import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerInNetworkQueryTransformer._ import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier @@ -17,7 +18,6 @@ import com.twitter.timelines.model.candidate.CandidateTweetSourceId object TimelineRankerInNetworkQueryTransformer { private val DefaultSinceDuration = 24.hours private val ExpandedSinceDuration = 48.hours - private val MaxTweetsToFetch = 600 private val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOption( includeReplies = true, @@ -39,14 +39,16 @@ object TimelineRankerInNetworkQueryTransformer { case class TimelineRankerInNetworkQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( - override val candidatePipelineIdentifier: CandidatePipelineIdentifier, - override val maxTweetsToFetch: Int = MaxTweetsToFetch) + override val candidatePipelineIdentifier: CandidatePipelineIdentifier) extends CandidatePipelineQueryTransformer[Query, t.RecapQuery] with TimelineRankerQueryTransformer[Query] { override val candidateTweetSourceId = CandidateTweetSourceId.RecycledTweet override val options = tweetKindOptions + override def maxTweetsToFetch(query: Query): Int = + query.params(FetchParams.InNetworkMaxTweetsToFetchParam) + override def getTensorflowModel(query: Query): Option[String] = { Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.InNetworkParam)) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala index e187a0aa0..b6d02b0b8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala @@ -2,6 +2,7 @@ package com.twitter.home_mixer.product.scored_tweets.query_transformer import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EarlybirdMaxResultsPerPartitionParam import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerQueryTransformer._ import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil @@ -32,16 +33,11 @@ object TimelineRankerQueryTransformer { * early-terminating the query and reducing the hits to MaxNumEarlybirdResults. */ private val EarlybirdMaxHits = 1000 - - /** - * Maximum number of results TLR should retrieve from each earlybird shard. - */ - private val EarlybirdMaxResults = 300 } trait TimelineRankerQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext] { - def maxTweetsToFetch: Int + def maxTweetsToFetch(query: Query): Int def options: TweetKindOption.ValueSet = TweetKindOption.Default def candidateTweetSourceId: CandidateTweetSourceId.Value def utegLikedByTweetsOptions(query: Query): Option[tlr.UtegLikedByTweetsOptions] = None @@ -68,9 +64,6 @@ trait TimelineRankerQueryTransformer[ untilTime) } - val maxCount = - (query.getQualityFactorCurrentValue(candidatePipelineIdentifier) * maxTweetsToFetch).toInt - val authorScoreMap = query.features .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[UserId, Double])) .getOrElse(Map.empty) @@ -82,7 +75,7 @@ trait TimelineRankerQueryTransformer[ val earlyBirdOptions = EarlybirdOptions( maxNumHitsPerShard = EarlybirdMaxHits, - maxNumResultsPerShard = EarlybirdMaxResults, + maxNumResultsPerShard = query.params(EarlybirdMaxResultsPerPartitionParam), models = earlybirdModels, authorScoreMap = authorScoreMap, skipVeryRecentTweets = true, @@ -91,7 +84,7 @@ trait TimelineRankerQueryTransformer[ tlr.RecapQuery( userId = query.getRequiredUserId, - maxCount = Some(maxCount), + maxCount = Some(maxTweetsToFetch(query)), range = Some(range), options = options, searchOperator = SearchOperator.Exclude, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala index ea051f331..b137116e1 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala @@ -4,6 +4,7 @@ import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FetchParams import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerUtegQueryTransformer._ import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer @@ -19,7 +20,6 @@ import com.twitter.timelines.model.candidate.CandidateTweetSourceId object TimelineRankerUtegQueryTransformer { private val SinceDuration = 24.hours - private val MaxTweetsToFetch = 300 private val MaxUtegCandidates = 800 private val tweetKindOptions = @@ -32,14 +32,16 @@ object TimelineRankerUtegQueryTransformer { case class TimelineRankerUtegQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( - override val candidatePipelineIdentifier: CandidatePipelineIdentifier, - override val maxTweetsToFetch: Int = MaxTweetsToFetch) + override val candidatePipelineIdentifier: CandidatePipelineIdentifier) extends CandidatePipelineQueryTransformer[Query, t.UtegLikedByTweetsQuery] with TimelineRankerQueryTransformer[Query] { override val candidateTweetSourceId = CandidateTweetSourceId.RecommendedTweet override val options = tweetKindOptions override val earlybirdModels = utegEarlybirdModels + + override def maxTweetsToFetch(query: Query): Int = + query.params(FetchParams.UTEGMaxTweetsToFetchParam) override def getTensorflowModel(query: Query): Option[String] = { Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.UtegParam)) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/UtegQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/UtegQueryTransformer.scala new file mode 100644 index 000000000..18d0fb470 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/UtegQueryTransformer.scala @@ -0,0 +1,68 @@ +package com.twitter.home_mixer.product.scored_tweets.query_transformer + +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature +import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.product.scored_tweets.query_transformer.UtegQueryTransformer._ +import com.twitter.home_mixer.util.CachedScoredTweetsHelper +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus +import com.twitter.recos.recos_common.thriftscala.SocialProofType +import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendationType +import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetEntityDisplayLocation +import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} +import com.twitter.util.Time + +object UtegQueryTransformer { + private val MaxUserSocialProofSize = 10 + private val MaxTweetSocialProofSize = 10 + private val MinUserSocialProofSize = 1 + private val DefaultSinceDuration = 24.hours + private val MaxTweetsToFetch = 800 + private val MaxExcludedTweets = 1500 +} + +case class UtegQueryTransformer[ + Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext +]( + candidatePipelineIdentifier: CandidatePipelineIdentifier) + extends CandidatePipelineQueryTransformer[Query, uteg.RecommendTweetEntityRequest] { + + override def transform(query: Query): uteg.RecommendTweetEntityRequest = { + + val weightedFollowings = query.features + .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double])) + .getOrElse(Map.empty) + + val sinceTime: Time = DefaultSinceDuration.ago + val untilTime: Time = Time.now + + val excludedTweetIds = query.features.map { featureMap => + CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweetsInRange( + featureMap, + candidatePipelineIdentifier, + MaxExcludedTweets, + sinceTime, + untilTime) + } + + uteg.RecommendTweetEntityRequest( + requesterId = query.getRequiredUserId, + displayLocation = TweetEntityDisplayLocation.HomeTimeline, + recommendationTypes = Seq(RecommendationType.Tweet), + seedsWithWeights = weightedFollowings, + maxResultsByType = Some(Map(RecommendationType.Tweet -> MaxTweetsToFetch)), + maxTweetAgeInMillis = Some(sinceTime.untilNow.inMillis), + excludedTweetIds = excludedTweetIds, + maxUserSocialProofSize = Some(MaxUserSocialProofSize), + maxTweetSocialProofSize = Some(MaxTweetSocialProofSize), + minUserSocialProofSizes = Some(Map(RecommendationType.Tweet -> MinUserSocialProofSize)), + socialProofTypes = Some(Seq(SocialProofType.Favorite)), + tweetAuthors = None, + maxEngagementAgeInMillis = None, + tweetTypes = None + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/BUILD.bazel index a337a328b..1360a1c82 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/BUILD.bazel @@ -3,20 +3,23 @@ scala_library( compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + "src/java/com/twitter/search/queryparser/query:core-query-nodes", + "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", "src/thrift/com/twitter/search:earlybird-scala", "timelines:util", "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", "timelines/src/main/scala/com/twitter/timelines/common/model", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", "timelines/src/main/scala/com/twitter/timelines/model/types", "timelines/src/main/scala/com/twitter/timelines/util/stats", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/CommunitiesEarlybirdQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/CommunitiesEarlybirdQueryTransformer.scala new file mode 100644 index 000000000..2956cbc6f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/CommunitiesEarlybirdQueryTransformer.scala @@ -0,0 +1,124 @@ +package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.thrift.ClientId +import com.twitter.finagle.tracing.Trace +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EarlyBirdCommunitiesMaxSearchResultsParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableEarlybirdCommunitiesQueryLinearRankingParam +import com.twitter.home_mixer.util.earlybird.RelevanceSearchUtil +import com.twitter.product_mixer.component_library.feature_hydrator.query.communities.CommunityMembershipsFeature +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.search.common.query.thriftjava.thriftscala.CollectorParams +import com.twitter.search.common.query.thriftjava.thriftscala.CollectorTerminationParams +import com.twitter.search.common.ranking.{thriftscala => scr} +import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.search.earlybird.{thriftscala => t} +import com.twitter.search.queryparser.query.Conjunction +import com.twitter.search.queryparser.query.Disjunction +import com.twitter.search.queryparser.query.search.SearchOperator +import com.twitter.search.queryparser.query.search.SearchOperatorConstants +import com.twitter.search.queryparser.query.{Query => SearchQuery} +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +@Singleton +class CommunitiesEarlybirdQueryTransformer @Inject() (clientId: ClientId) + extends CandidatePipelineQueryTransformer[PipelineQuery, eb.EarlybirdRequest] { + private val SinceDuration = 48.hours + private val DefaultSearchProcessingTimeout = 200.milliseconds + private val TensorflowModel = "timelines_unified_prod" + + override def transform(query: PipelineQuery): t.EarlybirdRequest = { + + val maxSearchResults = query.params(EarlyBirdCommunitiesMaxSearchResultsParam) + + val communityIds = + query.features.map(_.getOrElse(CommunityMembershipsFeature, Seq.empty)).toSeq.flatten.toSet + val entityIdsQuery = createEntityIdsQuery(communityIds) + val nullcastQuery = + new SearchOperator(SearchOperator.Type.INCLUDE, SearchOperatorConstants.NULLCAST) + val excludeRepliesQuery = + new SearchOperator(SearchOperator.Type.EXCLUDE, SearchOperatorConstants.REPLIES) + val sinceTimeQuery = + new SearchOperator(SearchOperator.Type.SINCE_TIME, SinceDuration.ago.inSeconds.toString) + val searchQuery = + new Conjunction(entityIdsQuery, excludeRepliesQuery, nullcastQuery, sinceTimeQuery) + + val metadataOptions = t.ThriftSearchResultMetadataOptions( + getResultLocation = false, + getLuceneScore = false, + getInReplyToStatusId = true, + getReferencedTweetAuthorId = true, + getMediaBits = true, + getAllFeatures = true, + returnSearchResultFeatures = true, + getFromUserId = true + ) + + val ebRankingParams = Some( + scr.ThriftRankingParams( + `type` = Some(scr.ThriftScoringFunctionType.TensorflowBased), + selectedTensorflowModel = Some(TensorflowModel), + minScore = -1.0e100, + applyBoosts = false, + ) + ) + + val linearRelevanceOptions = t.ThriftSearchRelevanceOptions( + rankingParams = Some( + scr.ThriftRankingParams( + `type` = Some(scr.ThriftScoringFunctionType.Linear), + applyBoosts = false, + favCountParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 1000.0)), + replyCountParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 10000.0)), + quotedCountParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 1000.0)) + ) + ), + returnAllResults = Some(false) + ) + + val relOptions = RelevanceSearchUtil.RelevanceOptions.copy( + rankingParams = ebRankingParams, + returnAllResults = Some(false) + ) + + val relevanceOptions = + if (query.params(EnableEarlybirdCommunitiesQueryLinearRankingParam)) linearRelevanceOptions + else relOptions + + val collectorParams = CollectorParams( + numResultsToReturn = maxSearchResults, + terminationParams = Some( + CollectorTerminationParams( + timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt + ) + ) + ) + + t.EarlybirdRequest( + searchQuery = t.ThriftSearchQuery( + serializedQuery = Some(searchQuery.serialize), + rankingMode = t.ThriftSearchRankingMode.Relevance, + numResults = maxSearchResults, + resultMetadataOptions = Some(metadataOptions), + searcherId = query.getOptionalUserId, + relevanceOptions = Some(relevanceOptions), + maxHitsPerUser = -1, + collectorParams = Some(collectorParams) + ), + clientRequestID = Some(s"${Trace.id.traceId}"), + numResultsToReturnAtRoot = Some(maxSearchResults), + clientId = Some(clientId.name), + ) + } + + private def createEntityIdsQuery(entityIds: Set[Long]): Disjunction = { + val entityIdStrings = entityIds.map(_.toString) + val queryOps: Seq[SearchQuery] = entityIdStrings.map { entityId => + new SearchOperator(SearchOperator.Type.ENTITY_ID, entityId) + }.toSeq + new Disjunction(queryOps.asJava) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdFrsQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdFrsQueryTransformer.scala index 9b2ac341e..521ff1332 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdFrsQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdFrsQueryTransformer.scala @@ -1,8 +1,11 @@ package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird import com.twitter.conversions.DurationOps._ +import com.twitter.core_workflows.user_model.{thriftscala => um} +import com.twitter.home_mixer.functional_component.feature_hydrator.FrsSeedUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUserIdsFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdFrsQueryTransformer._ import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier @@ -12,12 +15,21 @@ import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.timelines.common.model.TweetKindOption object EarlybirdFrsQueryTransformer { - private val SinceDuration = 24.hours - private val MaxTweetsToFetch = 100 - private val TensorflowModel = Some("timelines_rectweet_replica") + private val DefaultSinceDuration = 24.hours + private val ExpandedSinceDuration = 48.hours + private val MaxTweetsToFetch = 500 private val TweetKindOptions: TweetKindOption.ValueSet = TweetKindOption(includeOriginalTweetsAndQuotes = true) + + private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set( + um.UserState.Light, + um.UserState.MediumNonTweeter, + um.UserState.MediumTweeter, + um.UserState.NearZero, + um.UserState.New, + um.UserState.VeryLight + ) } case class EarlybirdFrsQueryTransformer[ @@ -30,12 +42,20 @@ case class EarlybirdFrsQueryTransformer[ override val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOptions override val maxTweetsToFetch: Int = MaxTweetsToFetch - override val tensorflowModel: Option[String] = TensorflowModel + override def getTensorflowModel(query: Query): Option[String] = { + Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.FrsParam)) + } override def transform(query: Query): eb.EarlybirdRequest = { + val userState = query.features.flatMap(_.getOrElse(UserStateFeature, None)) + val sinceDuration = + if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration + else DefaultSinceDuration + val seedUserIds = query.features .flatMap(_.getOrElse(FrsSeedUserIdsFeature, None)) .getOrElse(Seq.empty).toSet - buildEarlybirdQuery(query, SinceDuration, seedUserIds) + + buildEarlybirdQuery(query, sinceDuration, seedUserIds, None) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdInNetworkQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdInNetworkQueryTransformer.scala index 5c5464b8b..bd97aff24 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdInNetworkQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdInNetworkQueryTransformer.scala @@ -2,8 +2,11 @@ package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird import com.twitter.conversions.DurationOps._ import com.twitter.core_workflows.user_model.{thriftscala => um} +import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FollowedUserScoresFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdInNetworkQueryTransformer._ import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer @@ -16,10 +19,10 @@ import com.twitter.timelines.common.model.TweetKindOption object EarlybirdInNetworkQueryTransformer { private val DefaultSinceDuration = 24.hours private val ExpandedSinceDuration = 48.hours - private val MaxTweetsToFetch = 600 - private val TensorflowModel = Some("timelines_recap_replica") + private val MaxTweetsToFetch = 660 + private val MaxFollowUsers = 1500 - private val TweetKindOptions: TweetKindOption.ValueSet = TweetKindOption( + private val DefaultTweetKindOptions: TweetKindOption.ValueSet = TweetKindOption( includeReplies = true, includeRetweets = true, includeOriginalTweetsAndQuotes = true, @@ -44,25 +47,44 @@ case class EarlybirdInNetworkQueryTransformer[ extends CandidatePipelineQueryTransformer[Query, eb.EarlybirdRequest] with EarlybirdQueryTransformer[Query] { - override val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOptions + override def tweetKindOptions: TweetKindOption.ValueSet = DefaultTweetKindOptions override val maxTweetsToFetch: Int = MaxTweetsToFetch - override val tensorflowModel: Option[String] = TensorflowModel + override val enableExcludeSourceTweetIdsQuery = true + override def getTensorflowModel(query: Query): Option[String] = { + Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.InNetworkParam)) + } + + private def buildTweetKindOptions(query: Query): TweetKindOption.ValueSet = { + TweetKindOption( + includeReplies = query.params(ScoredTweetsParam.CandidateSourceParams.InNetworkIncludeRepliesParam), + includeRetweets = query.params(ScoredTweetsParam.CandidateSourceParams.InNetworkIncludeRetweetsParam), + includeOriginalTweetsAndQuotes = true, // Always include original tweets and quotes + includeExtendedReplies = query.params(ScoredTweetsParam.CandidateSourceParams.InNetworkIncludeExtendedRepliesParam) + ) + } override def transform(query: Query): eb.EarlybirdRequest = { - val userState = query.features.get.getOrElse(UserStateFeature, None) + val userState = query.features.flatMap(_.getOrElse(UserStateFeature, None)) val sinceDuration = if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration else DefaultSinceDuration - val followedUserIds = + val updatedAuthorScoreMap = query.features - .map( - _.getOrElse( - SGSFollowedUsersFeature, - Seq.empty)).toSeq.flatten.toSet + query.getRequiredUserId + .map(_.getOrElse(FollowedUserScoresFeature, Map.empty[Long, Double])).toSeq.flatten.toMap + val (authorScoreMap, followedUserIds) = if (updatedAuthorScoreMap.isEmpty) { + ( + query.features.map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double])), + query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSeq.flatten.toSet) + } else (Some(updatedAuthorScoreMap), updatedAuthorScoreMap.keySet) - buildEarlybirdQuery(query, sinceDuration, followedUserIds) + buildEarlybirdQueryWithTweetKindOptions( + query, + sinceDuration, + followedUserIds.take(MaxFollowUsers) + query.getRequiredUserId, + authorScoreMap, + buildTweetKindOptions(query)) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdQueryTransformer.scala index 0e51c54ea..a19a22890 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdQueryTransformer.scala @@ -1,9 +1,9 @@ package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird -import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus @@ -21,17 +21,53 @@ trait EarlybirdQueryTransformer[ def clientId: Option[String] = None def maxTweetsToFetch: Int = 100 def tweetKindOptions: TweetKindOption.ValueSet - def tensorflowModel: Option[String] = None + def getTensorflowModel(query: Query): Option[String] = None + def enableExcludeSourceTweetIdsQuery: Boolean = false private val EarlybirdMaxExcludedTweets = 1500 - def buildEarlybirdQuery( + protected def getFollowedUsers(query: Query): Set[Long] = { + query.features + .map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).getOrElse( + Nil).toSet + query.getRequiredUserId + } + + protected def buildEarlybirdQuery( + query: Query, + sinceDuration: Duration, + queryUserIds: Set[Long] = Set.empty, + authorScoreMap: Option[Map[Long, Double]] = None, + isVideoOnlyRequest: Boolean = false, + getOlderTweets: Boolean = false, + isRecency: Boolean = false, + until: Time = Time.now + ): eb.EarlybirdRequest = { + buildEarlybirdQueryWithTweetKindOptions( + query, + sinceDuration, + queryUserIds, + authorScoreMap, + tweetKindOptions, + isVideoOnlyRequest, + getOlderTweets, + isRecency, + until + ) + } + + protected def buildEarlybirdQueryWithTweetKindOptions( query: Query, sinceDuration: Duration, - followedUserIds: Set[Long] = Set.empty + queryUserIds: Set[Long] = Set.empty, + authorScoreMap: Option[Map[Long, Double]] = None, + tweetKindOptions: TweetKindOption.ValueSet, + isVideoOnlyRequest: Boolean = false, + getOlderTweets: Boolean = false, + isRecency: Boolean = false, + until: Time = Time.now ): eb.EarlybirdRequest = { val sinceTime: Time = sinceDuration.ago - val untilTime: Time = Time.now + val untilTime: Time = until val fromTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(sinceTime) val toTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(untilTime) @@ -45,26 +81,23 @@ trait EarlybirdQueryTransformer[ untilTime) } - val maxCount = - (query.getQualityFactorCurrentValue(candidatePipelineIdentifier) * maxTweetsToFetch).toInt - - val authorScoreMap = query.features - .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double])) - .getOrElse(Map.empty) - EarlybirdRequestUtil.getTweetsRequest( userId = Some(query.getRequiredUserId), clientId = clientId, skipVeryRecentTweets = true, - followedUserIds = followedUserIds, + queryUserIds = queryUserIds, retweetsMutedUserIds = Set.empty, beforeTweetIdExclusive = Some(toTweetIdExclusive), afterTweetIdExclusive = Some(fromTweetIdExclusive), excludedTweetIds = excludedTweetIds.map(_.toSet), - maxCount = maxCount, + maxCount = maxTweetsToFetch, tweetTypes = TweetTypes.fromTweetKindOption(tweetKindOptions), - authorScoreMap = Some(authorScoreMap), - tensorflowModel = tensorflowModel + authorScoreMap = authorScoreMap, + tensorflowModel = getTensorflowModel(query), + enableExcludeSourceTweetIdsQuery = enableExcludeSourceTweetIdsQuery, + isVideoOnlyRequest = isVideoOnlyRequest, + getOlderTweets = getOlderTweets, + isRecency = isRecency, ) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel index 30774b91f..b316938d2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel @@ -3,12 +3,16 @@ scala_library( compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ - "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", + "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "src/thrift/com/twitter/timelineranker:thrift-scala", "topic-social-proof/server/src/main/thrift:thrift-scala", "tweet-mixer/thrift/src/main/thrift:thrift-scala", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala index 0fbe7a438..0062a95e0 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala @@ -1,7 +1,9 @@ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.marshaller.timelines.TopicContextFunctionalityTypeUnmarshaller +import com.twitter.home_mixer.model.candidate_source.SourceSignal import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorFollowersFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature @@ -9,29 +11,64 @@ import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGoldVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGrayVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsLegacyVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.CachedCandidatePipelineIdentifierFeature +import com.twitter.home_mixer.model.HomeFeatures.ClipImageClusterIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.GorkContentCreatorFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokContentCreatorFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokSlopScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokTagsFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokTopCategoryFeature +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsArticleFeature import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.LastScoredTimestampMsFeature -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.ListIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ListNameFeature +import com.twitter.home_mixer.model.HomeFeatures.MultiModalEmbeddingsFeature +import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaClusterIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaCompletionRateFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMixerScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetTypeMetricsFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature +import com.twitter.home_mixer.model.HomeFeatures.ViralContentCreatorFeature import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature +import com.twitter.home_mixer.model.PredictedFavoriteScoreFeature +import com.twitter.home_mixer.model.PredictedReplyScoreFeature +import com.twitter.home_mixer.model.PredictedRetweetScoreFeature +import com.twitter.home_mixer.model.PredictedShareScoreFeature +import com.twitter.home_mixer.model.PredictedVideoQualityViewScoreFeature +import com.twitter.home_mixer.model.PredictedDwellScoreFeature +import com.twitter.home_mixer.model.PredictedNegativeFeedbackV2ScoreFeature +import com.twitter.home_mixer.model.PredictedGoodClickConvoDescUamGt2ScoreFeature +import com.twitter.home_mixer.model.PredictedGoodProfileClickScoreFeature +import com.twitter.home_mixer.model.PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature +import com.twitter.home_mixer.model.PredictedReplyEngagedByAuthorScoreFeature import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityNameFeature +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder @@ -52,8 +89,10 @@ object CachedScoredTweetsResponseFeatureTransformer AuthorIsGoldVerifiedFeature, AuthorIsGrayVerifiedFeature, AuthorIsLegacyVerifiedFeature, + AuthorFollowersFeature, CachedCandidatePipelineIdentifierFeature, DirectedAtUserIdFeature, + DebugStringFeature, ExclusiveConversationAuthorIdFeature, InNetworkFeature, InReplyToTweetIdFeature, @@ -61,7 +100,7 @@ object CachedScoredTweetsResponseFeatureTransformer IsReadFromCacheFeature, IsRetweetFeature, LastScoredTimestampMsFeature, - PerspectiveFilteredLikedByUserIdsFeature, + PredictionRequestIdFeature, QuotedTweetIdFeature, QuotedUserIdFeature, SGSValidFollowedByUserIdsFeature, @@ -69,15 +108,59 @@ object CachedScoredTweetsResponseFeatureTransformer ScoreFeature, SourceTweetIdFeature, SourceUserIdFeature, - StreamToKafkaFeature, - SuggestTypeFeature, + ServedTypeFeature, TopicContextFunctionalityTypeFeature, TopicIdSocialContextFeature, + TweetTypeMetricsFeature, TweetUrlsFeature, - WeightedModelScoreFeature + ViralContentCreatorFeature, + GrokContentCreatorFeature, + GorkContentCreatorFeature, + WeightedModelScoreFeature, + CommunityIdFeature, + CommunityNameFeature, + ListIdFeature, + ListNameFeature, + LocationIdFeature, + IsArticleFeature, + HasVideoFeature, + VideoDurationMsFeature, + TweetMediaIdsFeature, + GrokAnnotationsFeature, + GrokTopCategoryFeature, + GrokTagsFeature, + PredictedFavoriteScoreFeature, + PredictedReplyScoreFeature, + PredictedRetweetScoreFeature, + PredictedReplyEngagedByAuthorScoreFeature, + PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, + PredictedGoodClickConvoDescUamGt2ScoreFeature, + PredictedGoodProfileClickScoreFeature, + PredictedVideoQualityViewScoreFeature, + PredictedShareScoreFeature, + PredictedDwellScoreFeature, + PredictedNegativeFeedbackV2ScoreFeature, + TweetMixerScoreFeature, + SourceSignalFeature, + TweetMediaClusterIdsFeature, + ClipImageClusterIdsFeature, + GrokSlopScoreFeature, + TweetMediaCompletionRateFeature, + TweetTextFeature, + MultiModalEmbeddingsFeature ) - override def transform(candidate: hmt.ScoredTweet): FeatureMap = + override def transform(candidate: hmt.ScoredTweet): FeatureMap = { + val grokTopCategory = candidate.grokAnnotations + .flatMap(_.categoryScores) + .flatMap { scores => + val validCategories = scores.collect { + case (category, score) if category.forall(_.isDigit) && category.toLong % 10000 == 0 => + (category.toLong, score) + } + if (validCategories.nonEmpty) Some(validCategories.maxBy(_._2)._1) else None + } + FeatureMapBuilder() .add(AncestorsFeature, candidate.ancestors.getOrElse(Seq.empty)) .add(AuthorIdFeature, Some(candidate.authorId)) @@ -86,8 +169,10 @@ object CachedScoredTweetsResponseFeatureTransformer .add(AuthorIsGrayVerifiedFeature, candidate.authorMetadata.exists(_.grayVerified)) .add(AuthorIsLegacyVerifiedFeature, candidate.authorMetadata.exists(_.legacyVerified)) .add(AuthorIsCreatorFeature, candidate.authorMetadata.exists(_.creator)) + .add(AuthorFollowersFeature, candidate.authorMetadata.flatMap(_.followers)) .add(CachedCandidatePipelineIdentifierFeature, candidate.candidatePipelineIdentifier) .add(DirectedAtUserIdFeature, candidate.directedAtUserId) + .add(DebugStringFeature, candidate.debugString) .add(ExclusiveConversationAuthorIdFeature, candidate.exclusiveConversationAuthorId) .add(InNetworkFeature, candidate.inNetwork.getOrElse(true)) .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) @@ -95,9 +180,7 @@ object CachedScoredTweetsResponseFeatureTransformer .add(IsReadFromCacheFeature, true) .add(IsRetweetFeature, candidate.sourceTweetId.isDefined) .add(LastScoredTimestampMsFeature, candidate.lastScoredTimestampMs) - .add( - PerspectiveFilteredLikedByUserIdsFeature, - candidate.perspectiveFilteredLikedByUserIds.getOrElse(Seq.empty)) + .add(PredictionRequestIdFeature, candidate.predictionRequestId) .add(QuotedTweetIdFeature, candidate.quotedTweetId) .add(QuotedUserIdFeature, candidate.quotedUserId) .add(ScoreFeature, candidate.score) @@ -107,13 +190,79 @@ object CachedScoredTweetsResponseFeatureTransformer candidate.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) .add(SourceTweetIdFeature, candidate.sourceTweetId) .add(SourceUserIdFeature, candidate.sourceUserId) - .add(StreamToKafkaFeature, false) - .add(SuggestTypeFeature, candidate.suggestType) + .add(ServedTypeFeature, candidate.servedType) .add( TopicContextFunctionalityTypeFeature, candidate.topicFunctionalityType.map(TopicContextFunctionalityTypeUnmarshaller(_))) .add(TopicIdSocialContextFeature, candidate.topicId) + .add(TweetTypeMetricsFeature, candidate.tweetTypeMetrics) .add(TweetUrlsFeature, candidate.tweetUrls.getOrElse(Seq.empty)) + .add(ViralContentCreatorFeature, candidate.viralContentCreator.contains(true)) .add(WeightedModelScoreFeature, candidate.score) + .add(CommunityIdFeature, candidate.communityId) + .add(CommunityNameFeature, candidate.communityName) + .add(ListIdFeature, candidate.listId) + .add(ListNameFeature, candidate.listName) + .add(LocationIdFeature, candidate.locationId) + .add(IsArticleFeature, candidate.isArticle.contains(true)) + .add(HasVideoFeature, candidate.hasVideo.contains(true)) + .add(VideoDurationMsFeature, candidate.videoDurationMs) + .add(TweetMediaIdsFeature, candidate.mediaIds.getOrElse(Seq.empty)) + .add(GrokAnnotationsFeature, candidate.grokAnnotations) + .add(GrokTopCategoryFeature, grokTopCategory) + .add( + GrokTagsFeature, + candidate.grokAnnotations.map(_.tags.map(_.toLowerCase)).getOrElse(Seq.empty).toSet + ) + .add(PredictedFavoriteScoreFeature, candidate.predictedScores.flatMap(_.favoriteScore)) + .add(PredictedReplyScoreFeature, candidate.predictedScores.flatMap(_.replyScore)) + .add(PredictedRetweetScoreFeature, candidate.predictedScores.flatMap(_.retweetScore)) + .add( + PredictedReplyEngagedByAuthorScoreFeature, + candidate.predictedScores.flatMap(_.replyEngagedByAuthorScore)) + .add( + PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, + candidate.predictedScores.flatMap(_.goodClickConvoDescFavoritedOrRepliedScore)) + .add( + PredictedGoodClickConvoDescUamGt2ScoreFeature, + candidate.predictedScores.flatMap(_.goodClickConvoDescUamGt2Score)) + .add( + PredictedGoodProfileClickScoreFeature, + candidate.predictedScores.flatMap(_.goodProfileClickScore)) + .add( + PredictedVideoQualityViewScoreFeature, + candidate.predictedScores.flatMap(_.videoQualityViewScore)) + .add(PredictedShareScoreFeature, candidate.predictedScores.flatMap(_.shareScore)) + .add(PredictedDwellScoreFeature, candidate.predictedScores.flatMap(_.dwellScore)) + .add( + PredictedNegativeFeedbackV2ScoreFeature, + candidate.predictedScores.flatMap(_.negativeFeedbackV2Score)) + .add(TweetMixerScoreFeature, candidate.tweetMixerScore) + .add( + SourceSignalFeature, + candidate.sourceSignal.map { signal => + SourceSignal( + id = signal.id, + signalType = signal.signalType, + signalEntity = signal.signalEntity, + authorId = signal.authorId, + ) + } + ) + .add( + ClipImageClusterIdsFeature, + candidate.clipClusterIdsFeature + .flatMap(_.clipImageClusterIdsFeature).getOrElse(Map.empty[Long, Long]).toMap) + .add( + TweetMediaClusterIdsFeature, + candidate.clipClusterIdsFeature + .flatMap(_.tweetMediaClusterIdsFeature).getOrElse(Map.empty[Long, Long]).toMap) + .add(GrokSlopScoreFeature, candidate.grokSlopScoreFeature) + .add(TweetMediaCompletionRateFeature, candidate.mediaCompletionRate) + .add(TweetTextFeature, candidate.tweetText) + .add(GrokContentCreatorFeature, candidate.grokContentCreator.contains(true)) + .add(GorkContentCreatorFeature, candidate.gorkContentCreator.contains(true)) + .add(MultiModalEmbeddingsFeature, candidate.multiModalEmbedding) .build() + } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsBackfillResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsBackfillResponseFeatureTransformer.scala index 806c0d11d..7ce34d2f3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsBackfillResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsBackfillResponseFeatureTransformer.scala @@ -1,15 +1,14 @@ package com.twitter.home_mixer.product.scored_tweets.response_transformer -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} object ScoredTweetsBackfillResponseFeatureTransformer extends CandidateFeatureTransformer[Long] { @@ -17,14 +16,14 @@ object ScoredTweetsBackfillResponseFeatureTransformer extends CandidateFeatureTr TransformerIdentifier("ScoredTweetsBackfillResponse") override val features: Set[Feature[_, _]] = Set( - CandidateSourceIdFeature, FromInNetworkSourceFeature, - SuggestTypeFeature + ServedTypeFeature, + DebugStringFeature ) override def transform(candidate: Long): FeatureMap = FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.BackfillOrganicTweet)) .add(FromInNetworkSourceFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.RankedOrganicTweet)) + .add(ServedTypeFeature, hmt.ServedType.ForYouInNetwork) + .add(DebugStringFeature, Some("Backfill")) .build() } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsContentExplorationResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsContentExplorationResponseFeatureTransformer.scala new file mode 100644 index 000000000..466c23f70 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsContentExplorationResponseFeatureTransformer.scala @@ -0,0 +1,35 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer + +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.product.scored_tweets.candidate_source.ContentExplorationCandidateResponse +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier + +object ScoredTweetsContentExplorationResponseFeatureTransformer + extends CandidateFeatureTransformer[ContentExplorationCandidateResponse] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredTweetsContentExploration") + + override val features: Set[Feature[_, _]] = Set( + ServedTypeFeature, + DebugStringFeature + ) + + override def transform(candidate: ContentExplorationCandidateResponse): FeatureMap = { + val servedType = candidate.tier match { + case "tier1" => hmt.ServedType.ForYouContentExploration + case "tier2" => hmt.ServedType.ForYouContentExplorationTier2 + case _ => hmt.ServedType.ForYouContentExploration + } + FeatureMapBuilder() + .add(ServedTypeFeature, servedType) + .add(DebugStringFeature, Some(s"Content Exploration: ${candidate.tier}")) + .build() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsDirectUtegResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsDirectUtegResponseFeatureTransformer.scala new file mode 100644 index 000000000..3bfdd4266 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsDirectUtegResponseFeatureTransformer.scala @@ -0,0 +1,44 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer + +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.recos.recos_common.thriftscala.SocialProofType +import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} + +object UtegFavListFeature extends Feature[TweetCandidate, Seq[Long]] +object UtegScoreFeature extends Feature[TweetCandidate, Double] + +object ScoredTweetsDirectUtegResponseFeatureTransformer + extends CandidateFeatureTransformer[uteg.TweetRecommendation] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredTweetsDirectUtegResponse") + + override val features: Set[Feature[_, _]] = Set( + FromInNetworkSourceFeature, + ServedTypeFeature, + DebugStringFeature, + UtegFavListFeature, + UtegScoreFeature + ) + + override def transform(input: uteg.TweetRecommendation): FeatureMap = + FeatureMapBuilder() + .add(FromInNetworkSourceFeature, false) + .add(ServedTypeFeature, hmt.ServedType.ForYouUteg) + .add(DebugStringFeature, Some("Uteg")) + .add( + UtegFavListFeature, + input.socialProofByType.getOrElse(SocialProofType.Favorite, Seq.empty) + ) + .add(UtegScoreFeature, input.score) + .build() +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsFrsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsFrsResponseFeatureTransformer.scala index 077dccf24..82d62d4c7 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsFrsResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsFrsResponseFeatureTransformer.scala @@ -1,15 +1,13 @@ package com.twitter.home_mixer.product.scored_tweets.response_transformer -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.timelineranker.{thriftscala => tlr} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} object ScoredTweetsFrsResponseFeatureTransformer extends CandidateFeatureTransformer[tlr.CandidateTweet] { @@ -22,8 +20,7 @@ object ScoredTweetsFrsResponseFeatureTransformer val baseFeatures = TimelineRankerResponseTransformer.transform(candidate) val features = FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.FrsTweet)) - .add(SuggestTypeFeature, Some(st.SuggestType.FrsTweet)) + .add(ServedTypeFeature, hmt.ServedType.ForYouFrs) .build() baseFeatures ++ features diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala index d7d23bc98..63087550e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala @@ -1,16 +1,16 @@ package com.twitter.home_mixer.product.scored_tweets.response_transformer -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature +import com.twitter.home_mixer.model.candidate_source.SourceSignal import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.timelineranker.{thriftscala => tlr} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} object ScoredTweetsInNetworkResponseFeatureTransformer extends CandidateFeatureTransformer[tlr.CandidateTweet] { @@ -18,15 +18,25 @@ object ScoredTweetsInNetworkResponseFeatureTransformer override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsInNetworkResponse") - override val features: Set[Feature[_, _]] = TimelineRankerResponseTransformer.features + override val features: Set[Feature[_, _]] = + TimelineRankerResponseTransformer.features ++ Set(SourceSignalFeature) override def transform(candidate: tlr.CandidateTweet): FeatureMap = { val baseFeatures = TimelineRankerResponseTransformer.transform(candidate) val features = FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.RecycledTweet)) .add(FromInNetworkSourceFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.RankedTimelineTweet)) + .add(ServedTypeFeature, hmt.ServedType.ForYouInNetwork) + .add( + SourceSignalFeature, + Some( + SourceSignal( + id = candidate.tweet.flatMap(_.coreData.map(_.userId)).getOrElse(0L), + signalType = None, + signalEntity = None, + authorId = None, + )) + ) .build() baseFeatures ++ features diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsListsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsListsResponseFeatureTransformer.scala index 6a48e84c8..929fbe863 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsListsResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsListsResponseFeatureTransformer.scala @@ -1,45 +1,42 @@ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.ListIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.product.scored_tweets.candidate_source.ListTweet +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.{thriftscala => t} -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -object ScoredTweetsListsResponseFeatureTransformer extends CandidateFeatureTransformer[t.Tweet] { +object ScoredTweetsListsResponseFeatureTransformer extends CandidateFeatureTransformer[ListTweet] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsListsResponse") override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, - CandidateSourceIdFeature, FromInNetworkSourceFeature, IsRetweetFeature, - SuggestTypeFeature, + ServedTypeFeature, SourceTweetIdFeature, SourceUserIdFeature, + ListIdFeature ) - override def transform(candidate: t.Tweet): FeatureMap = { - FeatureMapBuilder() - .add(AuthorIdFeature, candidate.userId) - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.ListTweet)) - .add(FromInNetworkSourceFeature, false) - .add(IsRetweetFeature, candidate.sourceStatusId.isDefined) - .add(SuggestTypeFeature, Some(st.SuggestType.RankedListTweet)) - .add(SourceTweetIdFeature, candidate.sourceStatusId) - .add(SourceUserIdFeature, candidate.sourceUserId) - .build() - } + override def transform(candidate: ListTweet): FeatureMap = FeatureMapBuilder() + .add(AuthorIdFeature, candidate.tweet.userId) + .add(FromInNetworkSourceFeature, false) + .add(IsRetweetFeature, candidate.tweet.sourceStatusId.isDefined) + .add(ServedTypeFeature, hmt.ServedType.ForYouList) + .add(SourceTweetIdFeature, candidate.tweet.sourceStatusId) + .add(SourceUserIdFeature, candidate.tweet.sourceUserId) + .add(ListIdFeature, Some(candidate.listId)) + .build() } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsOfflineVideoRecoResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsOfflineVideoRecoResponseFeatureTransformer.scala new file mode 100644 index 000000000..a6ddef197 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsOfflineVideoRecoResponseFeatureTransformer.scala @@ -0,0 +1,27 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer + +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier + +object ScoredTweetsOfflineVideoRecoResponseFeatureTransformer + extends CandidateFeatureTransformer[Long] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredTweetsOfflineVideoRecoResponse") + + override val features: Set[Feature[_, _]] = Set( + FromInNetworkSourceFeature, + ServedTypeFeature + ) + + override def transform(candidate: Long): FeatureMap = FeatureMapBuilder() + .add(FromInNetworkSourceFeature, false) + .add(ServedTypeFeature, hmt.ServedType.OfflineVideoReco) + .build() +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsStaticResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsStaticResponseFeatureTransformer.scala new file mode 100644 index 000000000..dacfb4e50 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsStaticResponseFeatureTransformer.scala @@ -0,0 +1,33 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer + +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier + +object ScoredTweetsStaticResponseFeatureTransformer extends CandidateFeatureTransformer[Long] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredTweetsStatic") + + override val features: Set[Feature[_, _]] = Set( + FromInNetworkSourceFeature, + ServedTypeFeature, + ScoreFeature, + ) + + private val StaticScore = 100000.0 + + override def transform(candidate: Long): FeatureMap = { + FeatureMapBuilder() + .add(FromInNetworkSourceFeature, false) + .add(ServedTypeFeature, hmt.ServedType.ForYouTweetMixer) + .add(ScoreFeature, Some(StaticScore)) + .build() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsTweetMixerResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsTweetMixerResponseFeatureTransformer.scala index 4312b5104..948a5650f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsTweetMixerResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsTweetMixerResponseFeatureTransformer.scala @@ -1,52 +1,218 @@ package com.twitter.home_mixer.product.scored_tweets.response_transformer -import com.twitter.tweet_mixer.{thriftscala => tmt} import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.model.candidate_source._ +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} import com.twitter.tsp.{thriftscala => tsp} +import com.twitter.tweet_mixer.{thriftscala => tmt} +import com.twitter.usersignalservice.{thriftscala => uss} -object ScoredTweetsTweetMixerResponseFeatureTransformer +case class ScoredTweetsTweetMixerResponseFeatureTransformer(debugPrefix: String = "") extends CandidateFeatureTransformer[tmt.TweetResult] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsTweetMixerResponse") override val features: Set[Feature[_, _]] = Set( - CandidateSourceIdFeature, FromInNetworkSourceFeature, - IsRandomTweetFeature, - StreamToKafkaFeature, - SuggestTypeFeature, - TSPMetricTagFeature + ServedTypeFeature, + TSPMetricTagFeature, + TweetMixerScoreFeature, + InReplyToTweetIdFeature, + DebugStringFeature, + SourceSignalFeature, ) + val FavoriteSignal = "Fav" + val RetweetSignal = "Retweet" + val ReplySignal = "Reply" + val BookmarkSignal = "Bookmark" + val ShareSignal = "Share" + val TweetSignal = "Tweet" + val VideoViewSignal = "VideoView" + val ImmersiveVideoViewSignal = "ImmersiveVideoView" + val SearcherRealtimeHistorySignal = "SearcherRealtimeHistory" + val FollowSignal = "Follow" + val ProfileVisitSignal = "ProfileVisit" + val NotificationsSignal = "Notif" + + val UTEG = "UTEG" + val PopGeo = "PopGeo" + val PopTopic = "PopTopic" + val Simclusters = "Simclusters" + val Twhin = "Twhin" + val UTG = "UTG" + val UVG = "UVG" + val InNetwork = "InNetwork" + val DeepRetrieval = "DeepRetrieval" + val DeepRetrievalI2i = "DeepRetrievalI2i" + val ContentExploration = "Tier1ContentExploration" + val ContentExplorationTier2 = "Tier2ContentExploration" + val ContentExplorationDeepRetrievalI2i = "Tier1DrContentExploration" + val ContentExplorationTier2DeepRetrievalI2i = "Tier2DrContentExploration" + val ContentExplorationSimclusterColdPosts = "ContentExplorationSimclusterColdPosts" + val EvergreenDeepRetrievalHome = "EvergreenDeepRetrievalHome" + val EvergreenDeepRetrievalCrossBorderHome = "EvergreenDeepRetrievalCrossBorderHome" + val UserInterestSummary = "UserInterestSummary" + val ContentExplorationEvergreenDRI2i = "ContentExplorationEvergreenDRI2i" + val Local = "Local" + val Trends = "Trends" + val TwitterClipV0Short = "TwitterClipV0Short" + val TwitterClipV0Long = "TwitterClipV0Long" + val SemanticVideo = "SemanticVideo" + val RelatedCreator = "RelatedCreator" + override def transform(candidate: tmt.TweetResult): FeatureMap = { - val tweetMixerMetricTags = candidate.metricTags.getOrElse(Seq.empty) + val tweetMixerMetricTags: Seq[tmt.MetricTag] = candidate.metricTags.getOrElse(Seq.empty) val tspMetricTag = tweetMixerMetricTags - .map(TweetMixerMetricTagToTspMetricTag) + .map(tweetMixerMetricTagToTspMetricTag) .filter(_.nonEmpty).map(_.get).toSet + val servedType: hmt.ServedType = getServedType(candidate.tweetMetadata) + val fromInNetwork = servedType match { + case hmt.ServedType.ForYouInNetwork => true + case _ => false + } + FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.Simcluster)) - .add(FromInNetworkSourceFeature, false) - .add(IsRandomTweetFeature, false) - .add(StreamToKafkaFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.ScTweet)) + .add(FromInNetworkSourceFeature, fromInNetwork) + .add(ServedTypeFeature, servedType) .add(TSPMetricTagFeature, tspMetricTag) + .add(TweetMixerScoreFeature, candidate.score) + .add(DebugStringFeature, candidate.tweetMetadata.map(buildDebugString)) + .add(SourceSignalFeature, candidate.tweetMetadata.map(buildSourceSignal)) + .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) .build() } - private def TweetMixerMetricTagToTspMetricTag( + private def tweetMixerMetricTagToTspMetricTag( tweetMixerMetricTag: tmt.MetricTag ): Option[tsp.MetricTag] = tweetMixerMetricTag match { case tmt.MetricTag.TweetFavorite => Some(tsp.MetricTag.TweetFavorite) case tmt.MetricTag.Retweet => Some(tsp.MetricTag.Retweet) + case tmt.MetricTag.PopGeo => None case _ => None } + + private def buildSourceSignal(metadata: tmt.TweetMetadata): SourceSignal = { + SourceSignal( + id = metadata.sourceSignalId.getOrElse(0L), + signalType = metadata.signalType.flatMap(_.headOption.map(_.name)), + signalEntity = metadata.signalEntity, + authorId = metadata.authorId, + ) + } + + private def buildDebugString(metadata: tmt.TweetMetadata): String = { + val signalTypeStr = metadata.signalType + .map { signalTypes => + signalTypes.map { + case uss.SignalType.TweetFavorite => FavoriteSignal + case uss.SignalType.Retweet => RetweetSignal + case uss.SignalType.Reply => ReplySignal + case uss.SignalType.TweetBookmarkV1 => BookmarkSignal + case uss.SignalType.TweetShareV1 => ShareSignal + case uss.SignalType.OriginalTweet => TweetSignal + case uss.SignalType.VideoView90dQualityV1 | uss.SignalType.VideoView90dPlayback50V1 | + uss.SignalType.VideoView90dQualityV1AllSurfaces => + VideoViewSignal + case uss.SignalType.ImmersiveVideoQualityView => ImmersiveVideoViewSignal + case uss.SignalType.SearcherRealtimeHistory => SearcherRealtimeHistorySignal + case uss.SignalType.AccountFollow => FollowSignal + case uss.SignalType.RepeatedProfileVisit180dMinVisit6V1 | + uss.SignalType.RepeatedProfileVisit90dMinVisit6V1 | + uss.SignalType.RepeatedProfileVisit14dMinVisit2V1 | + uss.SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative | + uss.SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative | + uss.SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative => + ProfileVisitSignal + case uss.SignalType.NotificationOpenAndClickV1 => NotificationsSignal + case other => other.name + } + }.getOrElse(Seq.empty).mkString(",") + + val candidateTypeStr = metadata.servedType + .map { + case tmt.ServedType.Simclusters => Simclusters + case tmt.ServedType.Twhin => Twhin + case tmt.ServedType.Utg => UTG + case tmt.ServedType.Uvg => UVG + case tmt.ServedType.Uteg => UTEG + case tmt.ServedType.InNetwork => InNetwork + case tmt.ServedType.PopGeo => PopGeo + case tmt.ServedType.PopTopic => PopTopic + case tmt.ServedType.DeepRetrieval => DeepRetrieval + case tmt.ServedType.DeepRetrievalI2iEmb => DeepRetrievalI2i + case tmt.ServedType.ContentExploration => ContentExploration + case tmt.ServedType.ContentExplorationTier2 => ContentExplorationTier2 + case tmt.ServedType.ContentExplorationDRI2i => ContentExplorationDeepRetrievalI2i + case tmt.ServedType.ContentExplorationDRI2iTier2 => ContentExplorationTier2DeepRetrievalI2i + case tmt.ServedType.ContentExplorationSimclusterColdPosts => ContentExplorationSimclusterColdPosts + case tmt.ServedType.EvergreenDRU2iHome => EvergreenDeepRetrievalHome + case tmt.ServedType.EvergreenDRCrossBorderU2iHome => EvergreenDeepRetrievalCrossBorderHome + case tmt.ServedType.UserInterestSummaryI2i => UserInterestSummary + case tmt.ServedType.ContentExplorationEvergreenDRI2i => ContentExplorationEvergreenDRI2i + case tmt.ServedType.Local => Local + case tmt.ServedType.Trends => Trends + case tmt.ServedType.TwitterClipV0Short => TwitterClipV0Short + case tmt.ServedType.TwitterClipV0Long => TwitterClipV0Long + case tmt.ServedType.SemanticVideo => SemanticVideo + case tmt.ServedType.RelatedCreator => RelatedCreator + case _ => "" + }.getOrElse("") + + s"${metadata.sourceSignalId.getOrElse(0L)} $signalTypeStr $candidateTypeStr $debugPrefix" + } + + private def getServedType(metadata: Option[tmt.TweetMetadata]): hmt.ServedType = { + metadata + .flatMap { + _.servedType + .map { + case tmt.ServedType.Simclusters => hmt.ServedType.ForYouSimclusters + case tmt.ServedType.Twhin => hmt.ServedType.ForYouTwhin + case tmt.ServedType.Utg => hmt.ServedType.ForYouUtg + case tmt.ServedType.Uvg => hmt.ServedType.ForYouUvg + case tmt.ServedType.Uteg => hmt.ServedType.ForYouUteg + case tmt.ServedType.InNetwork => hmt.ServedType.ForYouInNetwork + case tmt.ServedType.PopGeo => hmt.ServedType.ForYouPopularGeo + case tmt.ServedType.PopTopic => hmt.ServedType.ForYouPopularTopic + case tmt.ServedType.DeepRetrieval => hmt.ServedType.ForYouDeepRetrieval + case tmt.ServedType.DeepRetrievalI2iEmb => hmt.ServedType.ForYouDeepRetrievalI2i + case tmt.ServedType.ContentExploration => hmt.ServedType.ForYouContentExploration + case tmt.ServedType.ContentExplorationTier2 => + hmt.ServedType.ForYouContentExplorationTier2 + case tmt.ServedType.ContentExplorationDRI2i => + hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i + case tmt.ServedType.ContentExplorationDRI2iTier2 => + hmt.ServedType.ForYouContentExplorationTier2DeepRetrievalI2i + case tmt.ServedType.EvergreenDeepRetrieval => + hmt.ServedType.ForYouEvergreenDeepRetrieval + case tmt.ServedType.EvergreenDRU2iHome => + hmt.ServedType.ForYouEvergreenDeepRetrievalHome + case tmt.ServedType.EvergreenDRCrossBorderU2iHome => + hmt.ServedType.ForYouEvergreenDeepRetrievalCrossBorderHome + case tmt.ServedType.UserInterestSummaryI2i => + hmt.ServedType.ForYouUserInterestSummary + case tmt.ServedType.ContentExplorationEvergreenDRI2i => + hmt.ServedType.ForYouContentExplorationEvergreenDeepRetrievalI2i + case tmt.ServedType.ContentExplorationSimclusterColdPosts => + hmt.ServedType.ForYouContentExplorationSimclusterColdPosts + case tmt.ServedType.Local => hmt.ServedType.ForYouLocal + case tmt.ServedType.Trends => hmt.ServedType.ForYouTrends + case tmt.ServedType.TwitterClipV0Short => hmt.ServedType.ForYouTwitterClipV0Short + case tmt.ServedType.TwitterClipV0Long => hmt.ServedType.ForYouTwitterClipV0Long + case tmt.ServedType.SemanticVideo => hmt.ServedType.ForYouSemanticVideo + case tmt.ServedType.RelatedCreator => hmt.ServedType.ForYouRelatedCreator + case tmt.ServedType.PromotedCreator => hmt.ServedType.ForYouPromotedCreator + case tmt.ServedType.NsfwVideoContent => hmt.ServedType.ForYouNsfwVideoContent + case _ => hmt.ServedType.ForYouTweetMixer + } + }.getOrElse(hmt.ServedType.ForYouTweetMixer) + } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredVideoTweetsPinnedTweetResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredVideoTweetsPinnedTweetResponseFeatureTransformer.scala new file mode 100644 index 000000000..85ff5743e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredVideoTweetsPinnedTweetResponseFeatureTransformer.scala @@ -0,0 +1,26 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer + +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier + +object ScoredVideoTweetsPinnedTweetResponseFeatureTransformer + extends CandidateFeatureTransformer[TweetCandidate] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredVideoTweetsPinnedTweetResponse") + + override val features: Set[Feature[_, _]] = Set( + ServedTypeFeature + ) + + override def transform(candidate: TweetCandidate): FeatureMap = FeatureMapBuilder() + .add(ServedTypeFeature, hmt.ServedType.ForYouPinned) + .build() + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala index a261b2fc2..09988573d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala @@ -1,7 +1,6 @@ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature @@ -11,18 +10,17 @@ import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder @@ -32,7 +30,7 @@ object TimelineRankerResponseTransformer { val features: Set[Feature[_, _]] = Set( AuthorIdFeature, - CandidateSourceIdFeature, + CommunityIdFeature, DirectedAtUserIdFeature, EarlybirdFeature, EarlybirdScoreFeature, @@ -42,16 +40,14 @@ object TimelineRankerResponseTransformer { HasVideoFeature, InReplyToTweetIdFeature, InReplyToUserIdFeature, - IsRandomTweetFeature, IsRetweetFeature, MentionScreenNameFeature, MentionUserIdFeature, - StreamToKafkaFeature, QuotedTweetIdFeature, QuotedUserIdFeature, SourceTweetIdFeature, SourceUserIdFeature, - SuggestTypeFeature, + ServedTypeFeature, TweetUrlsFeature ) @@ -62,9 +58,11 @@ object TimelineRankerResponseTransformer { val coreData = tweet.flatMap(_.coreData) val share = coreData.flatMap(_.share) val reply = coreData.flatMap(_.reply) + val communityId = tweet.flatMap(_.communities).flatMap(_.communityIds.headOption) FeatureMapBuilder() .add(AuthorIdFeature, coreData.map(_.userId)) + .add(CommunityIdFeature, communityId) .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) .add(EarlybirdFeature, candidate.features) .add(EarlybirdScoreFeature, candidate.features.map(_.earlybirdScore)) @@ -76,11 +74,9 @@ object TimelineRankerResponseTransformer { .add(HasVideoFeature, tweet.exists(TweetMediaFeaturesExtractor.hasVideo)) .add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId)) .add(InReplyToUserIdFeature, reply.map(_.inReplyToUserId)) - .add(IsRandomTweetFeature, candidate.features.exists(_.isRandomTweet.getOrElse(false))) .add(IsRetweetFeature, share.isDefined) .add(MentionScreenNameFeature, mentions.map(_.screenName)) .add(MentionUserIdFeature, mentions.flatMap(_.userId)) - .add(StreamToKafkaFeature, true) .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) .add(QuotedUserIdFeature, quotedTweet.map(_.userId)) .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/BUILD.bazel index 81cf19e9b..a89741fe2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/BUILD.bazel @@ -4,9 +4,8 @@ scala_library( strict_deps = True, dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", "src/thrift/com/twitter/search:earlybird-scala", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/EarlybirdResponseTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/EarlybirdResponseTransformer.scala index f0b1b59b1..bef93eb2f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/EarlybirdResponseTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/EarlybirdResponseTransformer.scala @@ -1,28 +1,11 @@ package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature -import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature -import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder @@ -31,59 +14,31 @@ import com.twitter.search.earlybird.{thriftscala => eb} object EarlybirdResponseTransformer { val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - CandidateSourceIdFeature, - DirectedAtUserIdFeature, EarlybirdScoreFeature, EarlybirdSearchResultFeature, - ExclusiveConversationAuthorIdFeature, - FromInNetworkSourceFeature, - HasImageFeature, - HasVideoFeature, InReplyToTweetIdFeature, InReplyToUserIdFeature, - IsRandomTweetFeature, IsRetweetFeature, - MentionScreenNameFeature, - MentionUserIdFeature, - StreamToKafkaFeature, - QuotedTweetIdFeature, - QuotedUserIdFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - SuggestTypeFeature, TweetUrlsFeature ) def transform(candidate: eb.ThriftSearchResult): FeatureMap = { - val tweet = candidate.tweetypieTweet - val quotedTweet = tweet.flatMap(_.quotedTweet) - val mentions = tweet.flatMap(_.mentions).getOrElse(Seq.empty) - val coreData = tweet.flatMap(_.coreData) - val share = coreData.flatMap(_.share) - val reply = coreData.flatMap(_.reply) + val metadata = candidate.metadata + val isRetweet = metadata.flatMap(_.isRetweet).getOrElse(false) + val sharedStatusId = metadata.map(_.sharedStatusId).getOrElse(0L) + val referencedTweetAuthorId = metadata.map(_.referencedTweetAuthorId).getOrElse(0L) + val inReplyToTweetId = if (!isRetweet && sharedStatusId > 0) Some(sharedStatusId) else None + val inReplyToUserId = + if (!isRetweet && sharedStatusId > 0 && referencedTweetAuthorId > 0) + Some(referencedTweetAuthorId) + else None + FeatureMapBuilder() - .add(AuthorIdFeature, coreData.map(_.userId)) - .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) .add(EarlybirdSearchResultFeature, Some(candidate)) .add(EarlybirdScoreFeature, candidate.metadata.flatMap(_.score)) - .add( - ExclusiveConversationAuthorIdFeature, - tweet.flatMap(_.exclusiveTweetControl.map(_.conversationAuthorId))) - .add(FromInNetworkSourceFeature, false) - .add(HasImageFeature, tweet.exists(TweetMediaFeaturesExtractor.hasImage)) - .add(HasVideoFeature, tweet.exists(TweetMediaFeaturesExtractor.hasVideo)) - .add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId)) - .add(InReplyToUserIdFeature, reply.map(_.inReplyToUserId)) - .add(IsRandomTweetFeature, candidate.tweetFeatures.exists(_.isRandomTweet.getOrElse(false))) - .add(IsRetweetFeature, share.isDefined) - .add(MentionScreenNameFeature, mentions.map(_.screenName)) - .add(MentionUserIdFeature, mentions.flatMap(_.userId)) - .add(StreamToKafkaFeature, true) - .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) - .add(QuotedUserIdFeature, quotedTweet.map(_.userId)) - .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) - .add(SourceUserIdFeature, share.map(_.sourceUserId)) + .add(InReplyToTweetIdFeature, inReplyToTweetId) + .add(InReplyToUserIdFeature, inReplyToUserId) + .add(IsRetweetFeature, isRetweet) .add( TweetUrlsFeature, candidate.metadata.flatMap(_.tweetUrls.map(_.map(_.originalUrl))).getOrElse(Seq.empty)) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsCommunitiesResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsCommunitiesResponseFeatureTransformer.scala new file mode 100644 index 000000000..ca718488c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsCommunitiesResponseFeatureTransformer.scala @@ -0,0 +1,42 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird + +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.search.earlybird.{thriftscala => t} + +object ScoredTweetsCommunitiesResponseFeatureTransformer + extends CandidateFeatureTransformer[t.ThriftSearchResult] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredTweetsEarlybirdCommunitiesResponse") + + private val candidateSourceFeatures = Set( + FromInNetworkSourceFeature, + ServedTypeFeature + ) + + override val features: Set[Feature[_, _]] = + EarlybirdResponseTransformer.features ++ Set(CommunityIdFeature) ++ candidateSourceFeatures + + override def transform(candidate: t.ThriftSearchResult): FeatureMap = { + val baseFeatures = EarlybirdResponseTransformer.transform(candidate) + + val communityIdOpt = + candidate.tweetypieTweet.flatMap(_.communities.flatMap(_.communityIds.headOption)) + + val features = FeatureMapBuilder() + .add(FromInNetworkSourceFeature, false) + .add(ServedTypeFeature, hmt.ServedType.ForYouCommunity) + .add(CommunityIdFeature, communityIdOpt) + .build() + + baseFeatures ++ features + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala index bb9ea8bee..89bfd0b64 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala @@ -1,15 +1,15 @@ package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} object ScoredTweetsEarlybirdFrsResponseFeatureTransformer extends CandidateFeatureTransformer[eb.ThriftSearchResult] { @@ -17,15 +17,22 @@ object ScoredTweetsEarlybirdFrsResponseFeatureTransformer override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsEarlybirdFrsResponse") - override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features + private val candidateSourceFeatures = Set( + FromInNetworkSourceFeature, + ServedTypeFeature, + DebugStringFeature + ) - override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { + override val features: Set[Feature[_, _]] = + EarlybirdResponseTransformer.features ++ candidateSourceFeatures + override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { val baseFeatures = EarlybirdResponseTransformer.transform(candidate) val features = FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.FrsTweet)) - .add(SuggestTypeFeature, Some(st.SuggestType.FrsTweet)) + .add(FromInNetworkSourceFeature, false) + .add(ServedTypeFeature, hmt.ServedType.ForYouFrs) + .add(DebugStringFeature, Some("FRS")) .build() baseFeatures ++ features diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala index 6b6a9d003..a1b34726c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala @@ -1,30 +1,38 @@ package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} object ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer extends CandidateFeatureTransformer[eb.ThriftSearchResult] { + override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsEarlybirdInNetworkResponse") - override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features + private val candidateSourceFeatures = Set( + FromInNetworkSourceFeature, + ServedTypeFeature, + DebugStringFeature + ) - override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { + override val features: Set[Feature[_, _]] = + EarlybirdResponseTransformer.features ++ candidateSourceFeatures + override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { val baseFeatures = EarlybirdResponseTransformer.transform(candidate) val features = FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.RecycledTweet)) - .add(SuggestTypeFeature, Some(st.SuggestType.RecycledTweet)) + .add(FromInNetworkSourceFeature, true) + .add(ServedTypeFeature, hmt.ServedType.ForYouInNetwork) + .add(DebugStringFeature, Some("In Network")) .build() baseFeatures ++ features diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdNsfwResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdNsfwResponseFeatureTransformer.scala new file mode 100644 index 000000000..004f52092 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdNsfwResponseFeatureTransformer.scala @@ -0,0 +1,40 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird + +import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.search.earlybird.{thriftscala => eb} + +object ScoredTweetsEarlybirdNsfwResponseFeatureTransformer + extends CandidateFeatureTransformer[eb.ThriftSearchResult] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredTweetsEarlybirdNsfwResponse") + + private val candidateSourceFeatures = Set( + FromInNetworkSourceFeature, + ServedTypeFeature, + DebugStringFeature + ) + + override val features: Set[Feature[_, _]] = + EarlybirdResponseTransformer.features ++ candidateSourceFeatures + + override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { + val baseFeatures = EarlybirdResponseTransformer.transform(candidate) + + val features = FeatureMapBuilder() + .add(FromInNetworkSourceFeature, false) + .add(ServedTypeFeature, hmt.ServedType.ForYouNsfwVideoContent) + .add(DebugStringFeature, Some("Nsfw Video")) + .build() + + baseFeatures ++ features + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/AuthorBasedListwiseRescoringProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/AuthorBasedListwiseRescoringProvider.scala new file mode 100644 index 000000000..c62da2f1c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/AuthorBasedListwiseRescoringProvider.scala @@ -0,0 +1,55 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object AuthorBasedListwiseRescoringProvider + extends ListwiseRescoringProvider[CandidateWithFeatures[TweetCandidate], Long] { + private val MinFollowed = 50 + + /** + * Author-based, the groupBy key is the author id. + * Rescore the list of candidates that share the same author id. + */ + override def groupByKey(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] = + candidate.features.getOrElse(AuthorIdFeature, None) + + /** + * Defines the list of author-based candidate rescorers. + * Multiply the rescorers together for each candidate. + */ + override def candidateRescoringFactor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate], + index: Int + ): Double = { + val isSmallFollowGraph = + query.features.get.getOrElse(SGSFollowedUsersFeature, Seq.empty).size <= MinFollowed + + val decayFactor = if (isSmallFollowGraph) { + query.params(ScoredTweetsParam.SmallFollowGraphAuthorDiversityDecayFactor) + } else { + query.params(ScoredTweetsParam.AuthorDiversityDecayFactor) + } + + val floor = + if (isSmallFollowGraph) query.params(ScoredTweetsParam.SmallFollowGraphAuthorDiversityFloor) + else query.params(ScoredTweetsParam.AuthorDiversityFloor) + + authorDiversityBasedRescorer(index = index, decayFactor = decayFactor, floor = floor) + } + + /** + * Re-scoring multiplier to apply to multiple tweets from the same author. + * Provides an exponential decay based discount by position (with a floor). + */ + def authorDiversityBasedRescorer( + index: Int, + decayFactor: Double, + floor: Double + ): Double = (1 - floor) * Math.pow(decayFactor, index) + floor +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel index 91ff12ee1..8ace67bf0 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel @@ -3,18 +3,21 @@ scala_library( compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/control_ai", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "src/scala/com/twitter/timelines/prediction/features/recap", + "src/thrift/com/twitter/timelines/control_ai:timeline-control-ai-thrift-scala", "timelineservice/common:model", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/CandidateSourceDiversityListwiseRescoringProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/CandidateSourceDiversityListwiseRescoringProvider.scala new file mode 100644 index 000000000..710578043 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/CandidateSourceDiversityListwiseRescoringProvider.scala @@ -0,0 +1,50 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature +import com.twitter.home_mixer.model.candidate_source.SourceSignal +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object CandidateSourceDiversityListwiseRescoringProvider + extends ListwiseRescoringProvider[ + CandidateWithFeatures[TweetCandidate], + (hmt.ServedType, Option[SourceSignal]) + ] { + + override def groupByKey( + candidate: CandidateWithFeatures[TweetCandidate] + ): Option[(hmt.ServedType, Option[SourceSignal])] = { + val servedType = candidate.features.get(ServedTypeFeature) + val sourceSignalOpt = candidate.features.getOrElse(SourceSignalFeature, None) + Some((servedType, sourceSignalOpt)) + } + + override def candidateRescoringFactor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate], + index: Int + ): Double = + candidate.features.get(ServedTypeFeature) match { + case hmt.ServedType.ForYouInNetwork => 1.0 + case _ => + if (query.params(ScoredTweetsParam.EnableCandidateSourceDiversityDecay)) { + val decayFactor = query.params(ScoredTweetsParam.CandidateSourceDiversityDecayFactor) + val floor = query.params(ScoredTweetsParam.CandidateSourceDiversityFloor) + candidateSourceDiversityRescorer(index = index, decayFactor = decayFactor, floor = floor) + } else 1.0 + } + + /** + * Re-scoring multiplier to apply to multiple tweets from the same candidate source and reason. + * Provides an exponential decay based discount by position (with a floor). + */ + def candidateSourceDiversityRescorer( + index: Int, + decayFactor: Double, + floor: Double + ): Double = (1 - floor) * Math.pow(decayFactor, index) + floor +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ContentExplorationListwiseRescoringProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ContentExplorationListwiseRescoringProvider.scala new file mode 100644 index 000000000..d28d69f58 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ContentExplorationListwiseRescoringProvider.scala @@ -0,0 +1,49 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationCandidateMaxCountParam +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object ContentExplorationListwiseRescoringProvider + extends ListwiseRescoringProvider[CandidateWithFeatures[TweetCandidate], hmt.ServedType] { + + override def groupByKey( + candidate: CandidateWithFeatures[TweetCandidate] + ): Option[hmt.ServedType] = + Some(candidate.features.get(ServedTypeFeature)) + + override def candidateRescoringFactor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate], + index: Int + ): Double = { + if (query.params(EnableContentExplorationCandidateMaxCountParam)) { + val servedType = candidate.features.get(ServedTypeFeature) + if (servedType == hmt.ServedType.ForYouUserInterestSummary || + servedType == hmt.ServedType.ForYouContentExploration || + servedType == hmt.ServedType.ForYouContentExplorationTier2 || + servedType == hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i || + servedType == hmt.ServedType.ForYouContentExplorationTier2DeepRetrievalI2i) 0.0001 + else 1.0 + } else 1.0 + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Map[Long, Double] = { + candidates + .groupBy(groupByKey) + .flatMap { + case (Some(servedType), groupedCandidates) => + groupedCandidates.zipWithIndex.map { + case (candidate, index) => + candidate.candidate.id -> candidateRescoringFactor(query, candidate, index) + } + case _ => Map.empty + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ControlAiRescorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ControlAiRescorer.scala new file mode 100644 index 000000000..7722f2975 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ControlAiRescorer.scala @@ -0,0 +1,55 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ControlAiEmbeddingSimilarityThresholdParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ControlAiShowLessScaleFactorParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ControlAiShowMoreScaleFactorParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableControlAiParam +import com.twitter.home_mixer.product.scored_tweets.util.ControlAiUtil +import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.ControlAiTopicEmbeddingMapFeature +import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.UserControlAiFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.control_ai.control.{thriftscala => ci} + +sealed trait ControlAiRescorer extends RescoringFactorProvider + +object ControlAiRescorer { + + private def buildControlAiRescorer( + actionType: ci.ActionType, + factorParam: FSBoundedParam[Double] + ): ControlAiRescorer = { + new ControlAiRescorer { + override def selector( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate] + ): Boolean = { + if (query.params(EnableControlAiParam)) { + val actions = query.features + .flatMap(_.getOrElse(UserControlAiFeature, None)) + .map(_.actions).getOrElse(Seq.empty).filter(_.actionType == actionType) + actions.exists( + ControlAiUtil.conditionMatch( + _, + candidate, + query.features.map(_.get(ControlAiTopicEmbeddingMapFeature)).getOrElse(Map.empty), + threshold = query.params(ControlAiEmbeddingSimilarityThresholdParam) + ) + ) + } else false + } + + override def factor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate] + ): Double = query.params(factorParam) + } + } + + val allRescorers = Seq( + buildControlAiRescorer(ci.ActionType.More, ControlAiShowMoreScaleFactorParam), + buildControlAiRescorer(ci.ActionType.Less, ControlAiShowLessScaleFactorParam) + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DeepRetrievalListwiseRescoringProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DeepRetrievalListwiseRescoringProvider.scala new file mode 100644 index 000000000..3d1caeae6 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DeepRetrievalListwiseRescoringProvider.scala @@ -0,0 +1,48 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.DeepRetrievalMaxCountParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableDeepRetrievalMaxCountParam +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object DeepRetrievalListwiseRescoringProvider + extends ListwiseRescoringProvider[CandidateWithFeatures[TweetCandidate], hmt.ServedType] { + + override def groupByKey( + candidate: CandidateWithFeatures[TweetCandidate] + ): Option[hmt.ServedType] = + Some(candidate.features.get(ServedTypeFeature)) + + override def candidateRescoringFactor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate], + index: Int + ): Double = { + if (query.params(EnableDeepRetrievalMaxCountParam)) { + val servedType = candidate.features.get(ServedTypeFeature) + val maxCount = query.params(DeepRetrievalMaxCountParam) + if (servedType == hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i && index >= maxCount) + 0.0001 + else 1.0 + } else 1.0 + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Map[Long, Double] = { + candidates + .groupBy(groupByKey) + .flatMap { + case (_, groupedCandidates) => + groupedCandidates.zipWithIndex.map { + case (candidate, index) => + candidate.candidate.id -> candidateRescoringFactor(query, candidate, index) + } + case _ => Map.empty + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/EvergreenDeepRetrievalCrossBorderListwiseRescoringProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/EvergreenDeepRetrievalCrossBorderListwiseRescoringProvider.scala new file mode 100644 index 000000000..a91310430 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/EvergreenDeepRetrievalCrossBorderListwiseRescoringProvider.scala @@ -0,0 +1,48 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EvergreenDeepRetrievalCrossBorderMaxCountParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableEvergreenDeepRetrievalCrossBorderMaxCountParam +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object EvergreenDeepRetrievalCrossBorderListwiseRescoringProvider + extends ListwiseRescoringProvider[CandidateWithFeatures[TweetCandidate], hmt.ServedType] { + + override def groupByKey( + candidate: CandidateWithFeatures[TweetCandidate] + ): Option[hmt.ServedType] = + Some(candidate.features.get(ServedTypeFeature)) + + override def candidateRescoringFactor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate], + index: Int + ): Double = { + if (query.params(EnableEvergreenDeepRetrievalCrossBorderMaxCountParam)) { + val servedType = candidate.features.get(ServedTypeFeature) + val maxCount = query.params(EvergreenDeepRetrievalCrossBorderMaxCountParam) + if (servedType == hmt.ServedType.ForYouEvergreenDeepRetrievalCrossBorderHome && index >= maxCount) + 0.0001 + else 1.0 + } else 1.0 + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Map[Long, Double] = { + candidates + .groupBy(groupByKey) + .flatMap { + case (_, groupedCandidates) => + groupedCandidates.zipWithIndex.map { + case (candidate, index) => + candidate.candidate.id -> candidateRescoringFactor(query, candidate, index) + } + case _ => Map.empty + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/EvergreenDeepRetrievalListwiseRescoringProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/EvergreenDeepRetrievalListwiseRescoringProvider.scala new file mode 100644 index 000000000..db586f9b9 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/EvergreenDeepRetrievalListwiseRescoringProvider.scala @@ -0,0 +1,48 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EvergreenDeepRetrievalMaxCountParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableEvergreenDeepRetrievalMaxCountParam +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object EvergreenDeepRetrievalListwiseRescoringProvider + extends ListwiseRescoringProvider[CandidateWithFeatures[TweetCandidate], hmt.ServedType] { + + override def groupByKey( + candidate: CandidateWithFeatures[TweetCandidate] + ): Option[hmt.ServedType] = + Some(candidate.features.get(ServedTypeFeature)) + + override def candidateRescoringFactor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate], + index: Int + ): Double = { + if (query.params(EnableEvergreenDeepRetrievalMaxCountParam)) { + val servedType = candidate.features.get(ServedTypeFeature) + val maxCount = query.params(EvergreenDeepRetrievalMaxCountParam) + if (servedType == hmt.ServedType.ForYouEvergreenDeepRetrievalHome && index >= maxCount) + 0.0001 + else 1.0 + } else 1.0 + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Map[Long, Double] = { + candidates + .groupBy(groupByKey) + .flatMap { + case (_, groupedCandidates) => + groupedCandidates.zipWithIndex.map { + case (candidate, index) => + candidate.candidate.id -> candidateRescoringFactor(query, candidate, index) + } + case _ => Map.empty + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/GrokSlopScoreRescorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/GrokSlopScoreRescorer.scala new file mode 100644 index 000000000..93800f1ad --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/GrokSlopScoreRescorer.scala @@ -0,0 +1,43 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.finagle.stats.DefaultStatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.GrokSlopScoreFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.GrokSlopScoreDecayValueParam +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object GrokSlopScoreRescorer { + + private val treatmentValue = 3L + + private val statsReceiver = DefaultStatsReceiver.scope("GrokSlopScoreRescorer") + private val numRescoredCandidatesCounter = statsReceiver.counter("rescored") + private val totalCandidatesCounter = statsReceiver.counter("total") + + private def onlyIf(query: PipelineQuery): Boolean = { + query.params(GrokSlopScoreDecayValueParam) < 1.0 + } + + def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Map[Long, Double] = { + if (!onlyIf(query)) { + return candidates.map { candidate => candidate.candidate.id -> 1.0 }.toMap + } + + val decayValue = query.params(GrokSlopScoreDecayValueParam) + + val rescoredCandidates = candidates.map { candidate => + val featureValue = candidate.features.getOrElse(GrokSlopScoreFeature, None) + val rescoreFactor = if (featureValue.contains(treatmentValue)) decayValue else 1.0 + candidate.candidate.id -> rescoreFactor + } + + numRescoredCandidatesCounter.incr(rescoredCandidates.count(_._2 != 1.0)) + totalCandidatesCounter.incr(candidates.size) + + rescoredCandidates.toMap + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HeuristicScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HeuristicScorer.scala index 765523b53..5f8a1e817 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HeuristicScorer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HeuristicScorer.scala @@ -1,44 +1,72 @@ package com.twitter.home_mixer.product.scored_tweets.scorer +import com.twitter.home_mixer.model.HomeFeatures.PhoenixScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.param.HomeGlobalParams.EnablePhoenixScorerParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableNoNegHeuristicParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnablePhoenixScoreParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MtlNormalization +import com.twitter.home_mixer.util.RerankerUtil.Epsilon import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch +import com.twitter.timelines.util.MtlNormalizer -/** - * Apply various heuristics to the model score - */ -object HeuristicScorer extends Scorer[ScoredTweetsQuery, TweetCandidate] { +object HeuristicScorer extends Scorer[PipelineQuery, TweetCandidate] { override val identifier: ScorerIdentifier = ScorerIdentifier("Heuristic") override val features: Set[Feature[_, _]] = Set(ScoreFeature) override def apply( - query: ScoredTweetsQuery, + query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { + val noNegHeuristic = query.params(EnableNoNegHeuristicParam) val rescorers = Seq( RescoreOutOfNetwork, RescoreReplies, - RescoreBlueVerified, - RescoreCreators, - RescoreMTLNormalization, - RescoreAuthorDiversity(AuthorDiversityDiscountProvider(candidates)), - RescoreFeedbackFatigue(query) - ) + RescoreMTLNormalization( + MtlNormalizer( + alpha = query.params(MtlNormalization.AlphaParam) / 100.0, + beta = query.params(MtlNormalization.BetaParam), + gamma = query.params(MtlNormalization.GammaParam) + ) + ), + RescoreListwise(ContentExplorationListwiseRescoringProvider(query, candidates)), + RescoreListwise(DeepRetrievalListwiseRescoringProvider(query, candidates)), + RescoreListwise(EvergreenDeepRetrievalListwiseRescoringProvider(query, candidates)), + RescoreListwise( + EvergreenDeepRetrievalCrossBorderListwiseRescoringProvider(query, candidates) + ), + RescoreListwise(AuthorBasedListwiseRescoringProvider(query, candidates)), + RescoreListwise(ImpressedAuthorDecayRescoringProvider(query, candidates)), + RescoreListwise(ImpressedMediaClusterBasedListwiseRescoringProvider(query, candidates)), + RescoreListwise(ImpressedImageClusterBasedListwiseRescoringProvider(query, candidates)), + RescoreListwise(CandidateSourceDiversityListwiseRescoringProvider(query, candidates)), + RescoreListwise(GrokSlopScoreRescorer(query, candidates)), + RescoreFeedbackFatigue(query), + RescoreListwise(MultimodalEmbeddingRescorer(query, candidates)), + RescoreLiveContent + ) ++ ControlAiRescorer.allRescorers + + val usePhoenix = query.params(EnablePhoenixScorerParam) && query.params(EnablePhoenixScoreParam) val updatedScores = candidates.map { candidate => - val score = candidate.features.getOrElse(ScoreFeature, None) + val scoreOpt = + if (usePhoenix) candidate.features.getOrElse(PhoenixScoreFeature, None) + else candidate.features.getOrElse(ScoreFeature, None) + val scaleFactor = rescorers.map(_(query, candidate)).product - val updatedScore = score.map(_ * scaleFactor) - FeatureMapBuilder().add(ScoreFeature, updatedScore).build() + val updatedScore = scoreOpt.map { score => + if (score < Epsilon && noNegHeuristic) score else score * scaleFactor + } + FeatureMap(ScoreFeature, updatedScore) } Stitch.value(updatedScores) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedAuthorDecayRescoringProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedAuthorDecayRescoringProvider.scala new file mode 100644 index 000000000..1699a6432 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedAuthorDecayRescoringProvider.scala @@ -0,0 +1,85 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedAuthorIdsFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableImpressionBasedAuthorDecay +import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweets +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object ImpressedAuthorDecayRescoringProvider { + + private def calculateAuthorImpressionFrequencies(query: PipelineQuery): Map[Long, Int] = { + val impressedTweetIds = + query.features.map(_.getOrElse(ImpressedTweets, Seq.empty)).getOrElse(Seq.empty).toSet + + val servedAuthorMap = query.features.map(_.get(ServedAuthorIdsFeature)).getOrElse(Map.empty) + + servedAuthorMap + .map { + case (authorId, tweetIds) => + val impressedCount = tweetIds.count(impressedTweetIds.contains) + authorId -> impressedCount + } + .filter(_._2 > 0) // Only include authors with at least one impressed tweet + } + + private def groupByKey(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] = + candidate.features.getOrElse(AuthorIdFeature, None) + + private def onlyIf(query: PipelineQuery): Boolean = query.params(EnableImpressionBasedAuthorDecay) + + def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Map[Long, Double] = { + if (onlyIf(query)) { + val outNetworkDecayFactor = + query.params(ScoredTweetsParam.AuthorDiversityOutNetworkDecayFactor) + val outNetworkFloor = query.params(ScoredTweetsParam.AuthorDiversityOutNetworkFloor) + + val inNetworkDecayFactor = query.params(ScoredTweetsParam.AuthorDiversityInNetworkDecayFactor) + val inNetworkFloor = query.params(ScoredTweetsParam.AuthorDiversityInNetworkFloor) + + val authorFreq = calculateAuthorImpressionFrequencies(query) + + candidates + .groupBy(groupByKey) + .flatMap { + case (Some(authorId), groupedCandidates) => + val sortedCandidates = groupedCandidates + .sortBy(_.features.getOrElse(ScoreFeature, None).getOrElse(0.0))( + Ordering.Double.reverse) + + sortedCandidates.zipWithIndex.map { + case (candidate, index) => + candidate.candidate.id -> { + val effectiveIndex = index + authorFreq.getOrElse(authorId, 0) + val isInNetworkCandidate = candidate.features.getOrElse(InNetworkFeature, true) + val decayFactor = + if (isInNetworkCandidate) inNetworkDecayFactor else outNetworkDecayFactor + val floor = if (isInNetworkCandidate) inNetworkFloor else outNetworkFloor + authorDiversityBasedRescorer(effectiveIndex, decayFactor, floor) + } + } + + case _ => Map.empty + } + } else Map.empty + } + + /** + * Re-scoring multiplier to apply to multiple tweets from the same author. + * Provides an exponential decay based discount by position (with a floor). + */ + private def authorDiversityBasedRescorer( + index: Int, + decayFactor: Double, + floor: Double + ): Double = (1 - floor) * Math.pow(decayFactor, index) + floor + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedImageClusterBasedListwiseRescoringProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedImageClusterBasedListwiseRescoringProvider.scala new file mode 100644 index 000000000..34116b9fb --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedImageClusterBasedListwiseRescoringProvider.scala @@ -0,0 +1,80 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.finagle.stats.DefaultStatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.ImpressedImageClusterIds +import com.twitter.home_mixer.model.HomeFeatures.ClipImageClusterIdsFeature +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ImpressedImageClusterBasedRescoringParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableImageClusterDecayParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableImageClusterFeatureHydrationParam +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object ImpressedImageClusterBasedListwiseRescoringProvider { + + /** + * Impressed Image cluster-Id based + * Rescore the list of candidates that have previously impressed imageCluster Ids. + */ + private val impressedImageStats = + DefaultStatsReceiver.scope("ImpressedImageClusterBasedListwiseRescoringProvider") + private val percentRescoredCandidatesStat = + impressedImageStats.stat("percent_rescored_candidates_x10") + + private def impressedImageClusterIds(query: PipelineQuery) = + query.features.map(_.getOrElse(ImpressedImageClusterIds, Seq[Long]())).getOrElse(Seq[Long]()) + + private def imageClusterId( + candidate: CandidateWithFeatures[TweetCandidate] + ) = candidate.features.getOrElse(ClipImageClusterIdsFeature, Map[Long, Long]()).values.toSeq + + private def rescoringFactor( + clusterIds: Seq[Long], + impressedImageClusterIdsToCountMap: Map[Long, Int], + decayFactor: Double + ): Double = { + val decayFactors = clusterIds.map { clusterId => + impressedImageClusterIdsToCountMap.get(clusterId) match { + case Some(count) => math.pow(1.0 - decayFactor, count) + case None => 1.0 + } + } + decayFactors.product + } + + private def onlyIf(query: PipelineQuery): Boolean = { + (query.params(ImpressedImageClusterBasedRescoringParam) > 0.0) && + query.params(EnableImageClusterDecayParam) && + query.params(EnableImageClusterFeatureHydrationParam) + } + + def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Map[Long, Double] = { + if (!onlyIf(query)) { + return candidates.map { candidate => + candidate.candidate.id -> 1.0 + }.toMap + } + + val impressedImageClusterIdsToCountMap = impressedImageClusterIds(query) + .groupBy(identity) + .mapValues(_.size) + + val decayFactor = query.params(ImpressedImageClusterBasedRescoringParam) + + val rescoringFactors = candidates.map { candidate => + val imageClusterIds = imageClusterId(candidate) + val rescoreFactor = + rescoringFactor(imageClusterIds, impressedImageClusterIdsToCountMap, decayFactor) + candidate.candidate.id -> rescoreFactor + }.toMap + + // Update Rescored candidates stats by number of candidates with rescoreFactor != 1 + percentRescoredCandidatesStat.add( + (rescoringFactors.count(_._2 != 1.0) * 1000.0 / (candidates.size + 0.01)).toFloat) + + rescoringFactors + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedMediaClusterBasedListwiseRescoringProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedMediaClusterBasedListwiseRescoringProvider.scala new file mode 100644 index 000000000..9b899b4fd --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedMediaClusterBasedListwiseRescoringProvider.scala @@ -0,0 +1,79 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.finagle.stats.DefaultStatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.ImpressedMediaClusterIds +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaClusterIdsFeature +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ImpressedMediaClusterBasedRescoringParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMediaClusterDecayParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMediaClusterFeatureHydrationParam +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object ImpressedMediaClusterBasedListwiseRescoringProvider { + + /** + * Impressed Media cluster-Id based + * Rescore the list of candidates that have previously impressed mediaCluster Ids. + */ + private val impressedMediaStats = + DefaultStatsReceiver.scope("ImpressedMediaClusterBasedListwiseRescoringProvider") + private val percentRescoredCandidatesStat = + impressedMediaStats.stat("percent_rescored_candidates_x10") + + private def impressedMediaClusterIds(query: PipelineQuery) = + query.features.map(_.getOrElse(ImpressedMediaClusterIds, Seq[Long]())).getOrElse(Seq[Long]()) + + private def mediaClusterId(candidate: CandidateWithFeatures[TweetCandidate]) = + candidate.features.getOrElse(TweetMediaClusterIdsFeature, Map[Long, Long]()).values.toSeq + + private def rescoringFactor( + clusterIds: Seq[Long], + impressedMediaClusterIdsToCountMap: Map[Long, Int], + decayFactor: Double + ): Double = { + val decayFactors = clusterIds.map { clusterId => + impressedMediaClusterIdsToCountMap.get(clusterId) match { + case Some(count) => math.pow(1.0 - decayFactor, count) + case None => 1.0 + } + } + decayFactors.product + } + + private def onlyIf(query: PipelineQuery): Boolean = { + (query.params(ImpressedMediaClusterBasedRescoringParam) > 0.0) && + query.params(EnableMediaClusterDecayParam) && + query.params(EnableMediaClusterFeatureHydrationParam) + } + + def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Map[Long, Double] = { + if (!onlyIf(query)) { + return candidates.map { candidate => + candidate.candidate.id -> 1.0 + }.toMap + } + + val impressedMediaClusterIdsToCountMap = impressedMediaClusterIds(query) + .groupBy(identity) + .mapValues(_.size) + + val decayFactor = query.params(ImpressedMediaClusterBasedRescoringParam) + + val rescoringFactors = candidates.map { candidate => + val mediaClusterIds = mediaClusterId(candidate) + val rescoreFactor = + rescoringFactor(mediaClusterIds, impressedMediaClusterIdsToCountMap, decayFactor) + candidate.candidate.id -> rescoreFactor + }.toMap + + // Update Rescored candidates stats by number of candidates with rescoreFactor != 1 + percentRescoredCandidatesStat.add( + (rescoringFactors.count(_._2 != 1.0) * 1000.0 / (candidates.size + 0.01)).toFloat) + + rescoringFactors + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ListwiseRescoringProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ListwiseRescoringProvider.scala new file mode 100644 index 000000000..593fac7c6 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ListwiseRescoringProvider.scala @@ -0,0 +1,54 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +/** + * Defines a listwise rescoring provider for use of rescoring a candidate dependent on the other candidates that + * co-exist with it. + * + * Requires: + * 1) groupByKey: How to define the set of candidates that are dependent on each other for scoring. + * 2) candidateRescoringFactor: Compute the final rescoring factor for each candidate in the group. + * + * This rescorer will sort the grouped candidates by their current score, and then apply the rescoring factors to + * each candidate in their group. + */ +trait ListwiseRescoringProvider[C <: CandidateWithFeatures[TweetCandidate], K] { + + /** + * Fetch the key used to create groups of candidates + */ + def groupByKey(candidate: C): Option[K] + + /** + * Compute the factor for each candidate based on position (zero-based) + * relative to other candidates associated with the same key + */ + def candidateRescoringFactor(query: PipelineQuery, candidate: C, index: Int): Double + + /** + * Group by the specified key (e.g. authors, likers, followers) + * Sort each group by score in descending order + * Determine the rescoring factor based on the position of each candidate + */ + def apply( + query: PipelineQuery, + candidates: Seq[C] + ): Map[Long, Double] = candidates + .groupBy(groupByKey) + .flatMap { + case (Some(_), groupedCandidates) => + val sortedCandidates = groupedCandidates + .sortBy(_.features.getOrElse(ScoreFeature, None).getOrElse(0.0))(Ordering.Double.reverse) + + sortedCandidates.zipWithIndex.map { + case (candidate, index) => + candidate.candidate.id -> candidateRescoringFactor(query, candidate, index) + } + + case _ => Map.empty + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/LowSignalScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/LowSignalScorer.scala new file mode 100644 index 000000000..ec8c1db55 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/LowSignalScorer.scala @@ -0,0 +1,60 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines +import com.twitter.product_mixer.core.model.common.presentation.CandidateSourcePosition +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import scala.collection.immutable.ListSet + +/** + * Assign scores based on candidate source positions, blending candidates from different sources + */ +object LowSignalScorer extends Scorer[PipelineQuery, TweetCandidate] { + + override val identifier: ScorerIdentifier = ScorerIdentifier("LowSignal") + + override val features: Set[Feature[_, _]] = Set(ScoreFeature) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val candidatesByPipeline = candidates.groupBy { + _.features.getOrElse(CandidatePipelines, ListSet.empty[CandidatePipelineIdentifier]).head + } + + val candidateScoreMap = candidatesByPipeline + .map { + case (_, pipelineCandidates) => + val sortedCandidates = pipelineCandidates.sortBy(_.features.get(CandidateSourcePosition)) + deduplicateAuthors(sortedCandidates).zipWithIndex.map { + case (candidate, index) => candidate.candidate.id -> index.toDouble + } + }.toSeq.flatten.toMap + + val maxScore = candidates.size.toDouble + val updatedScores = candidates.map { candidate => + val score = maxScore - candidateScoreMap.getOrElse(candidate.candidate.id, maxScore) + FeatureMap(ScoreFeature, Some(score)) + } + Stitch.value(updatedScores) + } + + def deduplicateAuthors( + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[CandidateWithFeatures[TweetCandidate]] = { + val seenAuthors = scala.collection.mutable.Set[Long]() + candidates.collect { + case c if seenAuthors.add(c.features.getOrElse(AuthorIdFeature, None).getOrElse(0L)) => c + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/MultimodalEmbeddingRescorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/MultimodalEmbeddingRescorer.scala new file mode 100644 index 000000000..6a5fb446c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/MultimodalEmbeddingRescorer.scala @@ -0,0 +1,88 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.finagle.stats.DefaultStatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.MultiModalEmbeddingsFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MultiModalEmbeddingRescorerGammaParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MultiModalEmbeddingRescorerMinScoreParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MultiModalEmbeddingRescorerNumCandidatesParam +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import scala.collection.mutable.ArrayBuffer + +object MultimodalEmbeddingRescorer { + + private val statsReceiver = DefaultStatsReceiver.scope("MultimodalEmbeddingRescorer") + private val rescoredCandidatesStat = statsReceiver.stat("rescored_candidates") + private val rescoreFactorx100Stat = statsReceiver.stat("rescore_factor_x100") + + private def onlyIf(query: PipelineQuery): Boolean = { + (query.params(MultiModalEmbeddingRescorerGammaParam) > 0.0) && + (query.params(MultiModalEmbeddingRescorerMinScoreParam) < 1.0) + } + + private def dot(a: Array[Double], b: Array[Double]): Double = { + a.zip(b).map { case (x, y) => x * y }.sum + } + + private def getSimilarityScoreFactors( + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + gamma: Double, + minScore: Double + ): Map[Long, Double] = { + val embsBuilder = ArrayBuffer.empty[Array[Double]] + val factorsBuilder = Map.newBuilder[Long, Double] + + candidates.foreach { candidate => + val id = candidate.candidate.id + candidate.features.getOrElse(MultiModalEmbeddingsFeature, None) match { + case Some(embSeq: Seq[Double]) => + val emb = embSeq.toArray + var similarCount = 0 + var i = 0 + while (i < embsBuilder.length) { + if (dot(embsBuilder(i), emb) > minScore) + similarCount += 1 + i += 1 + } + val factor = 1.0 / (1.0 + gamma * similarCount) + factorsBuilder += (id -> factor) + embsBuilder += emb + case _ => + } + } + + factorsBuilder.result() + } + + def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Map[Long, Double] = { + if (onlyIf(query)) { + // Get the top 100 candidates by score + val sortedCandidates = + candidates.sortBy(_.features.getOrElse(ScoreFeature, None).getOrElse(0.0)) + // Get top 100 and rest of candidates + val numCandidatesToScore = query.params(MultiModalEmbeddingRescorerNumCandidatesParam) + val topCandidates = sortedCandidates.take(numCandidatesToScore) + val restCandidates = sortedCandidates.drop(numCandidatesToScore) + + val candidateToFactorMap = getSimilarityScoreFactors( + topCandidates, + query.params(MultiModalEmbeddingRescorerGammaParam), + query.params(MultiModalEmbeddingRescorerMinScoreParam) + ) + val rescoredCandidates = candidateToFactorMap.filter { + case (_, factor) => factor != 1.0 + } + + rescoredCandidatesStat.add(rescoredCandidates.size.toFloat) + rescoreFactorx100Stat.add(rescoredCandidates.values.map(_.toFloat).sum * 100) + + // For the rest of the candidates, set factor to 1.0 + restCandidates.map(candidate => candidate.candidate.id -> 1.0).toMap ++ candidateToFactorMap + } else candidates.map(candidate => candidate.candidate.id -> 1.0).toMap + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/RescoringFactorProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/RescoringFactorProvider.scala index d9538b66d..146dee10f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/RescoringFactorProvider.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/RescoringFactorProvider.scala @@ -1,27 +1,21 @@ package com.twitter.home_mixer.product.scored_tweets.scorer +import com.twitter.home_mixer.functional_component.feature_hydrator.BroadcastStateFeature +import com.twitter.home_mixer.functional_component.feature_hydrator.SpaceStateFeature import com.twitter.home_mixer.functional_component.scorer.FeedbackFatigueScorer -import com.twitter.home_mixer.model.HomeFeatures -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature -import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.BlueVerifiedAuthorInNetworkMultiplierParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.BlueVerifiedAuthorOutOfNetworkMultiplierParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CreatorInNetworkMultiplierParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CreatorOutOfNetworkMultiplierParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.OutOfNetworkScaleFactorParam -import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam._ import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.util.MtlNormalizer import com.twitter.timelineservice.{thriftscala => tls} +import com.twitter.ubs.{thriftscala => ubs} trait RescoringFactorProvider { - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean + def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean def factor( query: PipelineQuery, @@ -31,43 +25,21 @@ trait RescoringFactorProvider { def apply( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate], - ): Double = if (selector(candidate)) factor(query, candidate) else 1.0 + ): Double = if (selector(query, candidate)) factor(query, candidate) else 1.0 } -/** - * Re-scoring multiplier to apply to authors who are eligible subscription content creators - */ -object RescoreCreators extends RescoringFactorProvider { - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = - candidate.features.getOrElse(AuthorIsCreatorFeature, false) && - CandidatesUtil.isOriginalTweet(candidate) +case class RescoreListwise(listwiseRescoringMap: Map[Long, Double]) + extends RescoringFactorProvider { - def factor( + override def selector( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] - ): Double = - if (candidate.features.getOrElse(InNetworkFeature, false)) - query.params(CreatorInNetworkMultiplierParam) - else query.params(CreatorOutOfNetworkMultiplierParam) -} - -/** - * Re-scoring multiplier to apply to authors who are verified by Twitter Blue - */ -object RescoreBlueVerified extends RescoringFactorProvider { + ): Boolean = listwiseRescoringMap.contains(candidate.candidate.id) - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = - candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false) && - CandidatesUtil.isOriginalTweet(candidate) - - def factor( + override def factor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] - ): Double = - if (candidate.features.getOrElse(InNetworkFeature, false)) - query.params(BlueVerifiedAuthorInNetworkMultiplierParam) - else query.params(BlueVerifiedAuthorOutOfNetworkMultiplierParam) + ): Double = listwiseRescoringMap(candidate.candidate.id) } /** @@ -75,7 +47,7 @@ object RescoreBlueVerified extends RescoringFactorProvider { */ object RescoreOutOfNetwork extends RescoringFactorProvider { - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = + def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean = !candidate.features.getOrElse(InNetworkFeature, false) def factor( @@ -89,59 +61,44 @@ object RescoreOutOfNetwork extends RescoringFactorProvider { */ object RescoreReplies extends RescoringFactorProvider { - private val ScaleFactor = 0.75 - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = + def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean = candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined def factor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] - ): Double = ScaleFactor + ): Double = query.params(ReplyScaleFactorParam) } /** * Re-scoring multiplier to calibrate multi-tasks learning model prediction */ -object RescoreMTLNormalization extends RescoringFactorProvider { +case class RescoreMTLNormalization(mtlNormalizer: MtlNormalizer) extends RescoringFactorProvider { - private val ScaleFactor = 1.0 - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = { - candidate.features.contains(HomeFeatures.FocalTweetAuthorIdFeature) - } + def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean = + query.params(MtlNormalization.EnableMtlNormalizationParam) def factor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] - ): Double = ScaleFactor -} - -/** - * Re-scoring multiplier to apply to multiple tweets from the same author - */ -case class RescoreAuthorDiversity(diversityDiscounts: Map[Long, Double]) - extends RescoringFactorProvider { - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = - diversityDiscounts.contains(candidate.candidate.id) - - def factor( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate] - ): Double = diversityDiscounts(candidate.candidate.id) + ): Double = mtlNormalizer( + attribute = candidate.features.getOrElse(AuthorFollowersFeature, None), + retweet = candidate.features.getOrElse(SourceTweetIdFeature, None).isDefined, + reply = candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined + ) } case class RescoreFeedbackFatigue(query: PipelineQuery) extends RescoringFactorProvider { - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = true + def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean = + true private val feedbackEntriesByEngagementType = query.features .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) .filter { entry => val timeSinceFeedback = query.queryTime.minus(entry.timestamp) - timeSinceFeedback < FeedbackFatigueScorer.DurationForFiltering + FeedbackFatigueScorer.DurationForDiscounting && + timeSinceFeedback < FeedbackFatigueScorer.DurationForDiscounting && entry.feedbackType == tls.FeedbackType.SeeFewer }.groupBy(_.engagementType) @@ -178,3 +135,25 @@ case class RescoreFeedbackFatigue(query: PipelineQuery) extends RescoringFactorP ) } } + +/** + * Disabled in production + */ +object RescoreLiveContent extends RescoringFactorProvider { + + private val MinFollowers = 1000000 + + def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean = { + ( + candidate.features.getOrElse(SpaceStateFeature, None).contains(ubs.BroadcastState.Running) || + candidate.features.getOrElse(BroadcastStateFeature, None).contains(ubs.BroadcastState.Running) + ) && + candidate.features.getOrElse(InNetworkFeature, false) && + candidate.features.getOrElse(AuthorFollowersFeature, None).exists(_ > MinFollowers) + } + + def factor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate] + ): Double = query.params(LiveContentScaleFactorParam) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel index a7fccd8ff..367e939e5 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel @@ -4,22 +4,30 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ + "home-mixer-features/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/embedding", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/real_time_aggregates", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", + "servo/repo/src/main/scala", + "src/scala/com/twitter/timelines/prediction/adapters/large_embeddings", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsHeuristicScoringPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsHeuristicScoringPipelineConfig.scala index aedfc15b5..88942a1fd 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsHeuristicScoringPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsHeuristicScoringPipelineConfig.scala @@ -1,14 +1,19 @@ package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsStaticCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.gate.DenyLowSignalUserGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.scorer.HeuristicScorer import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.core.functional_component.common.AllPipelines +import com.twitter.product_mixer.core.functional_component.common.AllExceptPipelines +import com.twitter.product_mixer.core.functional_component.gate.BaseGate import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig +import com.twitter.timelines.configapi.FSParam object ScoredTweetsHeuristicScoringPipelineConfig extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { @@ -16,7 +21,16 @@ object ScoredTweetsHeuristicScoringPipelineConfig override val identifier: ScoringPipelineIdentifier = ScoringPipelineIdentifier("ScoredTweetsHeuristic") - override val selectors: Seq[Selector[ScoredTweetsQuery]] = Seq(InsertAppendResults(AllPipelines)) + override val supportedClientParam: Option[FSParam[Boolean]] = + Some(ScoredTweetsParam.EnableHeuristicScoringPipeline) + + override val gates: Seq[BaseGate[ScoredTweetsQuery]] = Seq(DenyLowSignalUserGate) + + private val allExcept = AllExceptPipelines( + pipelinesToExclude = Set(ScoredTweetsStaticCandidatePipelineConfig.Identifier) + ) + + override val selectors: Seq[Selector[ScoredTweetsQuery]] = Seq(InsertAppendResults(allExcept)) override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq(HeuristicScorer) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsLowSignalScoringPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsLowSignalScoringPipelineConfig.scala new file mode 100644 index 000000000..9efbbbfe1 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsLowSignalScoringPipelineConfig.scala @@ -0,0 +1,26 @@ +package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline + +import com.twitter.home_mixer.product.scored_tweets.gate.AllowLowSignalUserGate +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.scorer.LowSignalScorer +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.selector.InsertAppendResults +import com.twitter.product_mixer.core.functional_component.common.AllPipelines +import com.twitter.product_mixer.core.functional_component.gate.BaseGate +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier +import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig + +object ScoredTweetsLowSignalScoringPipelineConfig + extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { + + override val identifier: ScoringPipelineIdentifier = + ScoringPipelineIdentifier("ScoredTweetsLowSignal") + + override val gates: Seq[BaseGate[ScoredTweetsQuery]] = Seq(AllowLowSignalUserGate) + + override val selectors: Seq[Selector[ScoredTweetsQuery]] = Seq(InsertAppendResults(AllPipelines)) + + override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq(LowSignalScorer) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsModelScoringPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsModelScoringPipelineConfig.scala index ab1b49a83..7990e9f6d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsModelScoringPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsModelScoringPipelineConfig.scala @@ -1,42 +1,48 @@ package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline import com.twitter.home_mixer.functional_component.feature_hydrator._ +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.TopicEdgeAggregateFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.TopicEdgeTruncatedAggregateFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.TweetContentEdgeAggregateFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.UserEngagerEdgeAggregateFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.UserEntityAggregateFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.MediaClusterIdFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates._ +import com.twitter.home_mixer.functional_component.scorer.NaviModelScorer +import com.twitter.home_mixer.functional_component.scorer.PhoenixScorer import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnablePhoenixScorerParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableGeoduckAuthorLocationHydatorParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableSimclustersSparseTweetFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTransformerPostEmbeddingJointBlueFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetLanguageFeaturesParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildTweetFeaturesOnlineParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinTweetFeaturesOnlineParam +import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableViewCountFeaturesParam import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsBackfillCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsFrsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsInNetworkCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsDirectUtegCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsListsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsPopularVideosCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsStaticCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsTweetMixerCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsUtegCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.AncestorFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.AuthorFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.AuthorIsCreatorFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird.ScoredTweetsCommunitiesCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird.ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.EarlybirdFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.GizmoduckAuthorFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.GraphTwoHopFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.MetricCenterUserCountingFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerAuthorFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerRelatedUsersFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealTimeInteractionGraphEdgeFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.SimClustersEngagementSimilarityFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.SimClustersUserTweetScoresHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TSPInferredTopicFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetMetaDataFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetTimeFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieContentFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TwhinAuthorFollowFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.UtegFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.Phase1EdgeAggregateFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.Phase2EdgeAggregateFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates._ +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ReplyFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.SemanticCoreFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.gate.DenyLowSignalUserGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableClipImagesClusterIdFeatureHydrationParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMediaCompletionRateFeatureHydrationParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMultiModalEmbeddingsFeatureHydratorParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableTweetTextV8EmbeddingFeatureParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.QualityFactor -import com.twitter.home_mixer.product.scored_tweets.scorer.NaviModelScorer import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.embedding.TweetTextV8EmbeddingFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.scorer.param_gated.ParamGatedScorer import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates @@ -55,69 +61,91 @@ import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandidateResult import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig import com.twitter.timelines.configapi.Param - import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsModelScoringPipelineConfig @Inject() ( - // candidate sources - scoredTweetsInNetworkCandidatePipelineConfig: ScoredTweetsInNetworkCandidatePipelineConfig, - scoredTweetsUtegCandidatePipelineConfig: ScoredTweetsUtegCandidatePipelineConfig, + // Candidate sources scoredTweetsTweetMixerCandidatePipelineConfig: ScoredTweetsTweetMixerCandidatePipelineConfig, - scoredTweetsFrsCandidatePipelineConfig: ScoredTweetsFrsCandidatePipelineConfig, scoredTweetsListsCandidatePipelineConfig: ScoredTweetsListsCandidatePipelineConfig, - scoredTweetsPopularVideosCandidatePipelineConfig: ScoredTweetsPopularVideosCandidatePipelineConfig, scoredTweetsBackfillCandidatePipelineConfig: ScoredTweetsBackfillCandidatePipelineConfig, - // feature hydrators + scoredTweetsEarlybirdInNetworkCandidatePipelineConfig: ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig, + scoredTweetsCommunitiesCandidatePipelineConfig: ScoredTweetsCommunitiesCandidatePipelineConfig, + scoredTweetsDirectUtegCandidatePipelineConfig: ScoredTweetsDirectUtegCandidatePipelineConfig, + // Feature hydrators ancestorFeatureHydrator: AncestorFeatureHydrator, authorFeatureHydrator: AuthorFeatureHydrator, - authorIsCreatorFeatureHydrator: AuthorIsCreatorFeatureHydrator, + broadcastStateFeatureHydrator: BroadcastStateFeatureHydrator, earlybirdFeatureHydrator: EarlybirdFeatureHydrator, - gizmoduckAuthorSafetyFeatureHydrator: GizmoduckAuthorFeatureHydrator, + gizmoduckAuthorFeatureHydrator: GizmoduckAuthorFeatureHydrator, + geoduckAuthorLocationHydrator: GeoduckAuthorLocationHydrator, graphTwoHopFeatureHydrator: GraphTwoHopFeatureHydrator, - metricCenterUserCountingFeatureHydrator: MetricCenterUserCountingFeatureHydrator, - perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator, + mediaClusterIdFeatureHydrator: MediaClusterIdFeatureHydrator, + mediaCompletionRateFeatureHydrator: MediaCompletionRateFeatureHydrator, + clipImagesClusterIdFeatureHydrator: ClipImageClusterIdFeatureHydrator, + viralContentCreatorMetricsFeatureHydrator: ViralContentCreatorMetricsFeatureHydrator, + multiModalEmbeddingsFeatureHydrator: MultiModalEmbeddingsFeatureHydrator, realGraphViewerAuthorFeatureHydrator: RealGraphViewerAuthorFeatureHydrator, realGraphViewerRelatedUsersFeatureHydrator: RealGraphViewerRelatedUsersFeatureHydrator, realTimeInteractionGraphEdgeFeatureHydrator: RealTimeInteractionGraphEdgeFeatureHydrator, + replyFeatureHydrator: ReplyFeatureHydrator, sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator, simClustersEngagementSimilarityFeatureHydrator: SimClustersEngagementSimilarityFeatureHydrator, simClustersUserTweetScoresHydrator: SimClustersUserTweetScoresHydrator, + spaceStateFeatureHydrator: SpaceStateFeatureHydrator, tspInferredTopicFeatureHydrator: TSPInferredTopicFeatureHydrator, - tweetypieContentFeatureHydrator: TweetypieContentFeatureHydrator, + tweetEntityServiceContentFeatureHydrator: TweetEntityServiceContentFeatureHydrator, + tweetTextV8EmbeddingFeatureHydrator: TweetTextV8EmbeddingFeatureHydrator, twhinAuthorFollowFeatureHydrator: TwhinAuthorFollowFeatureHydrator, + twhinTweetFeatureHydrator: TwhinTweetFeatureHydrator, + twhinRebuildTweetFeatureHydrator: TwhinRebuildTweetFeatureHydrator, utegFeatureHydrator: UtegFeatureHydrator, - // real time aggregate feature hydrators + slopAuthorFeatureHydrator: SlopAuthorFeatureHydrator, + grokAnnotationsFeatureHydrator: GrokAnnotationsFeatureHydrator, + // Real time aggregate feature hydrators engagementsReceivedByAuthorRealTimeAggregateFeatureHydrator: EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator, topicCountryEngagementRealTimeAggregateFeatureHydrator: TopicCountryEngagementRealTimeAggregateFeatureHydrator, topicEngagementRealTimeAggregateFeatureHydrator: TopicEngagementRealTimeAggregateFeatureHydrator, tweetCountryEngagementRealTimeAggregateFeatureHydrator: TweetCountryEngagementRealTimeAggregateFeatureHydrator, tweetEngagementRealTimeAggregateFeatureHydrator: TweetEngagementRealTimeAggregateFeatureHydrator, + tweetLanguageFeatureHydrator: TweetLanguageFeatureHydrator, twitterListEngagementRealTimeAggregateFeatureHydrator: TwitterListEngagementRealTimeAggregateFeatureHydrator, userAuthorEngagementRealTimeAggregateFeatureHydrator: UserAuthorEngagementRealTimeAggregateFeatureHydrator, - // offline aggregate feature hydrators - phase1EdgeAggregateFeatureHydrator: Phase1EdgeAggregateFeatureHydrator, - phase2EdgeAggregateFeatureHydrator: Phase2EdgeAggregateFeatureHydrator, - // model - naviModelScorer: NaviModelScorer) + viewCountsFeatureHydrator: ViewCountsFeatureHydrator, + // Large embeddings hydrators + authorLargeEmbeddingsFeatureHydrator: AuthorLargeEmbeddingsFeatureHydrator, + originalAuthorLargeEmbeddingsFeatureHydrator: OriginalAuthorLargeEmbeddingsFeatureHydrator, + tweetLargeEmbeddingsFeatureHydrator: TweetLargeEmbeddingsFeatureHydrator, + originalTweetLargeEmbeddingsFeatureHydrator: OriginalTweetLargeEmbeddingsFeatureHydrator, + // Transformer embeddings hydrators + transformerPostEmbeddingBlueFeatureHydrator: TransformerPostEmbeddingHomeBlueFeatureHydrator, + transformerPostEmbeddingGreenFeatureHydrator: TransformerPostEmbeddingHomeGreenFeatureHydrator, + transformerPostEmbeddingJointBlueFeatureHydrator: TransformerPostEmbeddingJointBlueFeatureHydrator, + simClustersLogFavBasedTweetFeatureHydrator: SimClustersLogFavBasedTweetFeatureHydrator, + // Scorers + naviModelScorer: NaviModelScorer, + phoenixScorer: PhoenixScorer) extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { override val identifier: ScoringPipelineIdentifier = ScoringPipelineIdentifier("ScoredTweetsModel") private val nonCachedScoringPipelineScope = AllExceptPipelines( - pipelinesToExclude = Set(CachedScoredTweetsCandidatePipelineConfig.Identifier) + pipelinesToExclude = Set( + CachedScoredTweetsCandidatePipelineConfig.Identifier, + ScoredTweetsStaticCandidatePipelineConfig.Identifier + ) ) override val gates: Seq[BaseGate[ScoredTweetsQuery]] = Seq( + DenyLowSignalUserGate, NonEmptyCandidatesGate(nonCachedScoringPipelineScope) ) private val earlybirdScorePipelineScope = Set( - scoredTweetsInNetworkCandidatePipelineConfig.identifier, - scoredTweetsUtegCandidatePipelineConfig.identifier, - scoredTweetsFrsCandidatePipelineConfig.identifier + scoredTweetsEarlybirdInNetworkCandidatePipelineConfig.identifier, + scoredTweetsDirectUtegCandidatePipelineConfig.identifier ) private val earlybirdScoreOrdering: Ordering[CandidateWithDetails] = @@ -134,8 +162,7 @@ class ScoredTweetsModelScoringPipelineConfig @Inject() ( new DropMaxCandidates( pipelineScope = SpecificPipelines(pipelineIdentifier), maxSelector = (query, _, _) => - (query.getQualityFactorCurrentValue(identifier) * - query.params(qualityFactorParam)).toInt + (query.getQualityFactorCurrentValue(identifier) * query.params(qualityFactorParam)).toInt ) } @@ -145,18 +172,6 @@ class ScoredTweetsModelScoringPipelineConfig @Inject() ( SpecificPipeline(scoredTweetsBackfillCandidatePipelineConfig.identifier), CandidatesUtil.reverseChronTweetsOrdering ), - qualityFactorDropMaxCandidates( - scoredTweetsInNetworkCandidatePipelineConfig.identifier, - QualityFactor.InNetworkMaxTweetsToScoreParam - ), - qualityFactorDropMaxCandidates( - scoredTweetsUtegCandidatePipelineConfig.identifier, - QualityFactor.UtegMaxTweetsToScoreParam - ), - qualityFactorDropMaxCandidates( - scoredTweetsFrsCandidatePipelineConfig.identifier, - QualityFactor.FrsMaxTweetsToScoreParam - ), qualityFactorDropMaxCandidates( scoredTweetsTweetMixerCandidatePipelineConfig.identifier, QualityFactor.TweetMixerMaxTweetsToScoreParam @@ -165,14 +180,22 @@ class ScoredTweetsModelScoringPipelineConfig @Inject() ( scoredTweetsListsCandidatePipelineConfig.identifier, QualityFactor.ListsMaxTweetsToScoreParam ), - qualityFactorDropMaxCandidates( - scoredTweetsPopularVideosCandidatePipelineConfig.identifier, - QualityFactor.PopularVideosMaxTweetsToScoreParam - ), qualityFactorDropMaxCandidates( scoredTweetsBackfillCandidatePipelineConfig.identifier, QualityFactor.BackfillMaxTweetsToScoreParam ), + qualityFactorDropMaxCandidates( + scoredTweetsEarlybirdInNetworkCandidatePipelineConfig.identifier, + QualityFactor.InNetworkMaxTweetsToScoreParam + ), + qualityFactorDropMaxCandidates( + scoredTweetsCommunitiesCandidatePipelineConfig.identifier, + QualityFactor.CommunitiesMaxTweetsToScoreParam + ), + qualityFactorDropMaxCandidates( + scoredTweetsDirectUtegCandidatePipelineConfig.identifier, + QualityFactor.UtegMaxTweetsToScoreParam + ), // Select candidates for Heavy Ranker Feature Hydration and Scoring InsertAppendResults(nonCachedScoringPipelineScope) ) @@ -180,44 +203,111 @@ class ScoredTweetsModelScoringPipelineConfig @Inject() ( override val preScoringFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq( - TweetMetaDataFeatureHydrator, - ancestorFeatureHydrator, + DependentBulkCandidateFeatureHydrator(ancestorFeatureHydrator, Seq(replyFeatureHydrator)), authorFeatureHydrator, - authorIsCreatorFeatureHydrator, - earlybirdFeatureHydrator, - gizmoduckAuthorSafetyFeatureHydrator, + DependentBulkCandidateFeatureHydrator( + earlybirdFeatureHydrator, + Seq(spaceStateFeatureHydrator, broadcastStateFeatureHydrator, TweetTimeFeatureHydrator)), + gizmoduckAuthorFeatureHydrator, + ParamGatedBulkCandidateFeatureHydrator( + EnableGeoduckAuthorLocationHydatorParam, + geoduckAuthorLocationHydrator, + ), graphTwoHopFeatureHydrator, - metricCenterUserCountingFeatureHydrator, - realTimeInteractionGraphEdgeFeatureHydrator, + InNetworkFeatureHydrator, realGraphViewerAuthorFeatureHydrator, + realTimeInteractionGraphEdgeFeatureHydrator, simClustersEngagementSimilarityFeatureHydrator, simClustersUserTweetScoresHydrator, - InNetworkFeatureHydrator, - tspInferredTopicFeatureHydrator, - tweetypieContentFeatureHydrator, + DependentBulkCandidateFeatureHydrator( + tspInferredTopicFeatureHydrator, + Seq( + TopicEdgeAggregateFeatureHydrator, + TopicEdgeTruncatedAggregateFeatureHydrator, + topicCountryEngagementRealTimeAggregateFeatureHydrator, + topicEngagementRealTimeAggregateFeatureHydrator + ) + ), + TweetMetaDataFeatureHydrator, + ParamGatedBulkCandidateFeatureHydrator( + EnableTweetTextV8EmbeddingFeatureParam, + tweetTextV8EmbeddingFeatureHydrator + ), + DependentBulkCandidateFeatureHydrator( + tweetEntityServiceContentFeatureHydrator, + Seq( + SemanticCoreFeatureHydrator, + TweetContentEdgeAggregateFeatureHydrator, + mediaClusterIdFeatureHydrator)), twhinAuthorFollowFeatureHydrator, - utegFeatureHydrator, - // real time aggregates + ParamGatedBulkCandidateFeatureHydrator( + EnableTwhinTweetFeaturesOnlineParam, + twhinTweetFeatureHydrator + ), + ParamGatedBulkCandidateFeatureHydrator( + EnableTwhinRebuildTweetFeaturesOnlineParam, + twhinRebuildTweetFeatureHydrator + ), + ParamGatedBulkCandidateFeatureHydrator( + EnableViewCountFeaturesParam, + viewCountsFeatureHydrator + ), + DependentBulkCandidateFeatureHydrator( + utegFeatureHydrator, + Seq( + realGraphViewerRelatedUsersFeatureHydrator, + sgsValidSocialContextFeatureHydrator, + UserEngagerEdgeAggregateFeatureHydrator + ) + ), + slopAuthorFeatureHydrator, + // Real time aggregates engagementsReceivedByAuthorRealTimeAggregateFeatureHydrator, tweetCountryEngagementRealTimeAggregateFeatureHydrator, tweetEngagementRealTimeAggregateFeatureHydrator, + ParamGatedBulkCandidateFeatureHydrator( + EnableTweetLanguageFeaturesParam, + tweetLanguageFeatureHydrator + ), twitterListEngagementRealTimeAggregateFeatureHydrator, userAuthorEngagementRealTimeAggregateFeatureHydrator, - // offline aggregates - phase1EdgeAggregateFeatureHydrator + // Offline aggregates + UserEntityAggregateFeatureHydrator, + viralContentCreatorMetricsFeatureHydrator, + GrokGorkContentCreatorFeatureHydrator, + // Large Embeddings + authorLargeEmbeddingsFeatureHydrator, + originalAuthorLargeEmbeddingsFeatureHydrator, + tweetLargeEmbeddingsFeatureHydrator, + originalTweetLargeEmbeddingsFeatureHydrator, + // Transformers + transformerPostEmbeddingBlueFeatureHydrator, + transformerPostEmbeddingGreenFeatureHydrator, + ParamGatedBulkCandidateFeatureHydrator( + EnableTransformerPostEmbeddingJointBlueFeaturesParam, + transformerPostEmbeddingJointBlueFeatureHydrator + ), + ParamGatedBulkCandidateFeatureHydrator( + EnableSimclustersSparseTweetFeaturesParam, + simClustersLogFavBasedTweetFeatureHydrator + ), + grokAnnotationsFeatureHydrator, + ParamGatedBulkCandidateFeatureHydrator( + EnableMediaCompletionRateFeatureHydrationParam, + mediaCompletionRateFeatureHydrator, + ), + ParamGatedBulkCandidateFeatureHydrator( + EnableClipImagesClusterIdFeatureHydrationParam, + clipImagesClusterIdFeatureHydrator + ), + ParamGatedBulkCandidateFeatureHydrator( + EnableMultiModalEmbeddingsFeatureHydratorParam, + multiModalEmbeddingsFeatureHydrator + ) ) - override val preScoringFeatureHydrationPhase2: Seq[ - BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq( - perspectiveFilteredSocialContextFeatureHydrator, - phase2EdgeAggregateFeatureHydrator, - realGraphViewerRelatedUsersFeatureHydrator, - sgsValidSocialContextFeatureHydrator, - TweetTimeFeatureHydrator, - topicCountryEngagementRealTimeAggregateFeatureHydrator, - topicEngagementRealTimeAggregateFeatureHydrator + override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq( + naviModelScorer, + ParamGatedScorer(EnablePhoenixScorerParam, phoenixScorer) ) - - override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq(naviModelScorer) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsRerankingScoringPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsRerankingScoringPipelineConfig.scala new file mode 100644 index 000000000..b63b8fe99 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsRerankingScoringPipelineConfig.scala @@ -0,0 +1,40 @@ +package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline + +import com.twitter.home_mixer.functional_component.scorer.PhoenixModelRerankingScorer +import com.twitter.home_mixer.functional_component.scorer.WeighedModelRerankingScorer +import com.twitter.home_mixer.param.HomeGlobalParams.EnablePhoenixScorerParam +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.scorer.param_gated.ParamGatedScorer +import com.twitter.product_mixer.core.functional_component.gate.BaseGate +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier +import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ScoredTweetsRerankingScoringPipelineConfig @Inject() ( + // rerankers + weighedModelRerankingScorer: WeighedModelRerankingScorer, + phoenixModelRerankingScorer: PhoenixModelRerankingScorer, + // Base scoring pipeline config + scoredTweetsModelScoringPipelineConfig: ScoredTweetsModelScoringPipelineConfig) + extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { + + override val identifier: ScoringPipelineIdentifier = + ScoringPipelineIdentifier("ScoredTweetsReranking") + + override val gates: Seq[BaseGate[ScoredTweetsQuery]] = + scoredTweetsModelScoringPipelineConfig.gates + + override val selectors: Seq[Selector[ScoredTweetsQuery]] = + scoredTweetsModelScoringPipelineConfig.selectors + + override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = + Seq( + weighedModelRerankingScorer, + ParamGatedScorer(EnablePhoenixScorerParam, phoenixModelRerankingScorer) + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/BUILD.bazel index c5dbb187f..162295c89 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/BUILD.bazel @@ -5,8 +5,9 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepTopKCandidatesPerCommunity.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepTopKCandidatesPerCommunity.scala new file mode 100644 index 000000000..2667983ee --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepTopKCandidatesPerCommunity.scala @@ -0,0 +1,37 @@ +package com.twitter.home_mixer.product.scored_tweets.selector + +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +case class KeepTopKCandidatesPerCommunity(override val pipelineScope: CandidateScope) + extends Selector[PipelineQuery] { + + private val MaxCandidatesPerCommunity = 1 + private val MaxCommunityCandidates = 3 + + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val (selectedCandidates, otherCandidates) = remainingCandidates.partition { candidate => + pipelineScope.contains(candidate) && + candidate.features.getOrElse(CommunityIdFeature, None).isDefined + } + + val filteredCandidates = selectedCandidates + .groupBy { candidate => candidate.features.getOrElse(CommunityIdFeature, None) } + .values.flatMap { + _.sortBy(_.features.getOrElse(ScoreFeature, None)).reverse.take(MaxCandidatesPerCommunity) + } + .toSeq.sortBy(_.features.getOrElse(ScoreFeature, None)).reverse.take(MaxCommunityCandidates) + + val updatedCandidates = otherCandidates ++ filteredCandidates + SelectorResult(remainingCandidates = updatedCandidates, result = result) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel index 2147ee217..16c65e23e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel @@ -6,28 +6,44 @@ scala_library( dependencies = [ "finagle/finagle-mysql/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", + "kafka/finagle-kafka/finatra-kafka/src/main/scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweet_mixer", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", "servo/repo/src/main/scala", - "servo/util/src/main/scala", - "src/scala/com/twitter/timelines/prediction/common/adapters", + "src/scala/com/twitter/timelines/prediction/common/adapters:base", + "src/scala/com/twitter/timelines/prediction/features/common", + "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", "src/thrift/com/twitter/timelines/suggests/common:data_record_metadata-scala", "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", + "strato/config/columns/home-mixer:home-mixer-strato-client", + "strato/config/columns/videoRecommendations/twitterClip:twitterClip-strato-client", + "strato/config/src/thrift/com/twitter/strato/columns/content_understanding:content_understanding-scala", + "timelines/data_processing/jobs/light_ranking/light_ranking_features_prep:recap_partial_features_for_two_tower_models", + "timelines/ml:kafka", "timelines/ml:pldr-client", "timelines/ml:pldr-conversion", "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", - "timelines/src/main/scala/com/twitter/timelines/util/stats", + "user_history_transformer/service/src/main/java/com/x/user_action_sequence", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheCandidateFeaturesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheCandidateFeaturesSideEffect.scala new file mode 100644 index 000000000..e6fccba4a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheCandidateFeaturesSideEffect.scala @@ -0,0 +1,45 @@ +package com.twitter.home_mixer.product.scored_tweets.side_effect +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.side_effect.BaseCacheCandidateFeaturesSideEffect +import com.twitter.home_mixer.param.HomeMixerFlagName.DataRecordMetadataStoreConfigsYmlFlag +import com.twitter.home_mixer.param.HomeMixerInjectionNames._ +import com.twitter.inject.annotations.Flag +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.storehaus.ReadableStore +import com.twitter.storehaus.Store +import com.twitter.strato.generated.client.videoRecommendations.twitterClip.TwitterClipEmbeddingMhClientColumn +import com.twitter.timelines.served_candidates_logging.{thriftscala => sc} +import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pdr} +import com.twitter.twistly.thriftscala.VideoViewEngagementType +import com.twitter.twistly.thriftscala.WatchTimeMetadata +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class CacheCandidateFeaturesSideEffect @Inject() ( + @Flag(DataRecordMetadataStoreConfigsYmlFlag) dataRecordMetadataStoreConfigsYml: String, + @Named(MemcacheCandidateFeaturesStore) store: Store[ + sc.CandidateFeatureKey, + pdr.PolyDataRecord + ], + @Named(TweetWatchTimeMetadataStore) tweetWatchTimeMetadataStore: ReadableStore[ + (Long, VideoViewEngagementType), + WatchTimeMetadata + ], + twitterClipEmbeddingMhClientColumn: TwitterClipEmbeddingMhClientColumn, + @Named(TwhinVideoEmbeddingsStore) twhinVideoStore: ReadableStore[Long, TwhinTweetEmbedding], + statsReceiver: StatsReceiver) + extends BaseCacheCandidateFeaturesSideEffect( + dataRecordMetadataStoreConfigsYml, + store, + tweetWatchTimeMetadataStore, + twitterClipEmbeddingMhClientColumn, + twhinVideoStore, + statsReceiver) { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("CacheCandidateFeatures") + + override val statScope: String = getClass.getSimpleName +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheRequestInfoSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheRequestInfoSideEffect.scala new file mode 100644 index 000000000..4d5eaa820 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheRequestInfoSideEffect.scala @@ -0,0 +1,73 @@ +package com.twitter.home_mixer.product.scored_tweets.side_effect + +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature +import com.twitter.home_mixer.model.PredictedFavoriteScoreFeature +import com.twitter.home_mixer.model.PredictedReplyScoreFeature +import com.twitter.home_mixer.model.PredictedRetweetScoreFeature +import com.twitter.home_mixer.model.PredictedShareScoreFeature +import com.twitter.home_mixer.model.PredictedVideoQualityViewScoreFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableCacheRequestInfoParam +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.HasMarshalling +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Client +import com.twitter.strato.generated.client.home_mixer.StoreRequestInfoClientColumn +import com.twitter.strato.generated.client.home_mixer.StoreRequestInfoClientColumn.FavAndRetweetAndReplyAndShareAndVqv +import com.twitter.strato.generated.client.home_mixer.StoreRequestInfoClientColumn.PostIdAndHeavyRankerScores +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class CacheRequestInfoSideEffect @Inject() ( + @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client, + serviceIdentifier: ServiceIdentifier) + extends PipelineResultSideEffect[PipelineQuery, HasMarshalling] + with Conditionally[PipelineQuery, HasMarshalling] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("CacheRequestInfo") + + private val isProdEnv = serviceIdentifier.environment == "prod" + + private val retrievalSignalExecutor = new StoreRequestInfoClientColumn(stratoClient).executer + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Boolean = query.params(EnableCacheRequestInfoParam) && isProdEnv + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, HasMarshalling] + ): Stitch[Unit] = { + val posts = inputs.selectedCandidates.collect { + case candidate if !candidate.features.getOrElse(IsReadFromCacheFeature, false) => + PostIdAndHeavyRankerScores( + postId = candidate.candidateIdLong, + heavyRankerScores = Some( + FavAndRetweetAndReplyAndShareAndVqv( + fav = candidate.features.getOrElse(PredictedFavoriteScoreFeature, None), + retweet = candidate.features.getOrElse(PredictedRetweetScoreFeature, None), + reply = candidate.features.getOrElse(PredictedReplyScoreFeature, None), + share = candidate.features.getOrElse(PredictedShareScoreFeature, None), + vqv = candidate.features.getOrElse(PredictedVideoQualityViewScoreFeature, None) + ) + ) + ) + } + val arg = StoreRequestInfoClientColumn.Arg( + userId = inputs.query.getRequiredUserId, + posts = posts, + requestTimestampMs = inputs.query.queryTime.inMilliseconds + ) + retrievalSignalExecutor.execute(arg) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheRetrievalSignalSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheRetrievalSignalSideEffect.scala new file mode 100644 index 000000000..ebc918567 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheRetrievalSignalSideEffect.scala @@ -0,0 +1,116 @@ +package com.twitter.home_mixer.product.scored_tweets.side_effect + +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableCacheRetrievalSignalParam +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.HasMarshalling +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Client +import com.twitter.strato.client.Putter +import com.twitter.strato.generated.client.home_mixer.RetrievalSignalv2ClientColumn +import com.twitter.usersignalservice.{thriftscala => se} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class CacheRetrievalSignalSideEffect @Inject() ( + @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client, + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver) + extends PipelineResultSideEffect[PipelineQuery, HasMarshalling] + with Conditionally[PipelineQuery, HasMarshalling] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("CacheRetrievalSignal") + + private val scopedStats = statsReceiver.scope(getClass.getSimpleName) + + private val isProdEnv = serviceIdentifier.environment == "prod" + + private val retrievalSignalPutter: Putter[(Long, Long), hmt.RetrievalSignal] = + new RetrievalSignalv2ClientColumn(stratoClient).putter + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Boolean = query.params(EnableCacheRetrievalSignalParam) && isProdEnv + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, HasMarshalling] + ): Stitch[Unit] = { + scopedStats.counter("side_effect_applied").incr(1) + + val sourceSignalWrites = inputs.selectedCandidates.map { candidate => + scopedStats.counter("total_candidates_processed").incr(1) + + val userId = inputs.query.getRequiredUserId + val signalFeature = candidate.features.getOrElse(SourceSignalFeature, None) + val servedType = candidate.features.get(ServedTypeFeature) + + val retrievalSignalOpt = (signalFeature, servedType) match { + case (Some(signal), _) if signal.id > 0L && signal.signalEntity.isDefined => + scopedStats.counter("valid_existing_signal_feature").incr(1) + signal.signalEntity.flatMap { entity => + if (entity != se.SignalEntity.User || signal.id != userId) { + Some( + hmt.RetrievalSignal( + signalId = signal.id, + signalEntity = entity, + authorId = signal.authorId.filter(_ > 0L) + )) + } else None + } + + case (None, hmt.ServedType.ForYouInNetwork) => + scopedStats.counter("valid_in_network_signal_feature").incr(1) + candidate.features.get(AuthorIdFeature).map { authorId => + hmt.RetrievalSignal( + signalId = authorId, + signalEntity = se.SignalEntity.User, + authorId = None + ) + } + + case (None, hmt.ServedType.ForYouUteg) => + scopedStats.counter("valid_uteg_liked_by_signal_feature").incr(1) + val validLikedByUserIds = + candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) + validLikedByUserIds.headOption.map { lastLikedByUserId => + hmt.RetrievalSignal( + signalId = lastLikedByUserId, + signalEntity = se.SignalEntity.User, + authorId = None + ) + } + + case _ => + scopedStats.counter("missing_or_invalid_signal_feature").incr(1) + None + } + retrievalSignalOpt + .map { retrievalSignal => + val sourceTweetId: Long = + candidate.features + .getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidateIdLong) + retrievalSignalPutter.put((userId, sourceTweetId), retrievalSignal) + }.getOrElse(Stitch.Unit) + }.toSeq + + Stitch.collect(sourceSignalWrites).map(_ => ()) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala index 3d66ff54a..033b74662 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala @@ -1,55 +1,61 @@ package com.twitter.home_mixer.product.scored_tweets.side_effect -import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGoldVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGrayVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsLegacyVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.CachedCandidatePipelineIdentifierFeature -import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.LastScoredTimestampMsFeature -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature -import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.transport.Transport +import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.model.PredictedFavoriteScoreFeature +import com.twitter.home_mixer.model.PredictedReplyScoreFeature +import com.twitter.home_mixer.model.PredictedRetweetScoreFeature +import com.twitter.home_mixer.model.PredictedShareScoreFeature +import com.twitter.home_mixer.model.PredictedVideoQualityViewScoreFeature +import com.twitter.home_mixer.model.PredictedDwellScoreFeature +import com.twitter.home_mixer.model.PredictedNegativeFeedbackV2ScoreFeature +import com.twitter.home_mixer.model.PredictedGoodClickConvoDescUamGt2ScoreFeature +import com.twitter.home_mixer.model.PredictedGoodProfileClickScoreFeature +import com.twitter.home_mixer.model.PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature +import com.twitter.home_mixer.model.PredictedReplyEngagedByAuthorScoreFeature +import com.twitter.home_mixer.param.HomeMixerInjectionNames.ScoredTweetsCache import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature +import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityNameFeature +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.TopicContextFunctionalityTypeMarshaller import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.TtlCache import com.twitter.stitch.Stitch import javax.inject.Inject +import javax.inject.Named import javax.inject.Singleton @Singleton class CachedScoredTweetsSideEffect @Inject() ( + @Named(ScoredTweetsCache) scoredTweetsCache: TtlCache[Long, hmt.ScoredTweetsResponse]) - extends PipelineResultSideEffect[PipelineQuery, ScoredTweetsResponse] { + extends PipelineResultSideEffect[PipelineQuery, ScoredTweetsResponse] + with Conditionally[PipelineQuery, ScoredTweetsResponse] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("CachedScoredTweets") private val MaxTweetsToCache = 1000 + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: ScoredTweetsResponse + ): Boolean = { + val serviceIdentifier = ServiceIdentifier.fromCertificate(Transport.peerCertificate) + serviceIdentifier.role != "explore-mixer" && serviceIdentifier.role != "video-mixer" + } + def buildCachedScoredTweets( query: PipelineQuery, candidates: Seq[CandidateWithDetails] @@ -57,18 +63,47 @@ class CachedScoredTweetsSideEffect @Inject() ( val tweets = candidates.map { candidate => val sgsValidLikedByUserIds = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) + val validLikedByUserIds = candidate.features.getOrElse(ValidLikedByUserIdsFeature, Seq.empty) val sgsValidFollowedByUserIds = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) - val perspectiveFilteredLikedByUserIds = - candidate.features.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty) val ancestors = candidate.features.getOrElse(AncestorsFeature, Seq.empty) + val mediaIds = candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty) + val sourceSignalOpt = candidate.features.getOrElse(SourceSignalFeature, None) + val sourceSignal = sourceSignalOpt.map { signal => + hmt.SourceSignal( + id = signal.id, + signalType = signal.signalType, + signalEntity = signal.signalEntity, + authorId = signal.authorId, + ) + } + + val predictedScores = hmt.PredictedScores( + favoriteScore = candidate.features.getOrElse(PredictedFavoriteScoreFeature, None), + replyScore = candidate.features.getOrElse(PredictedReplyScoreFeature, None), + retweetScore = candidate.features.getOrElse(PredictedRetweetScoreFeature, None), + replyEngagedByAuthorScore = + candidate.features.getOrElse(PredictedReplyEngagedByAuthorScoreFeature, None), + goodClickConvoDescFavoritedOrRepliedScore = candidate.features + .getOrElse(PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, None), + goodClickConvoDescUamGt2Score = + candidate.features.getOrElse(PredictedGoodClickConvoDescUamGt2ScoreFeature, None), + goodProfileClickScore = + candidate.features.getOrElse(PredictedGoodProfileClickScoreFeature, None), + videoQualityViewScore = + candidate.features.getOrElse(PredictedVideoQualityViewScoreFeature, None), + shareScore = candidate.features.getOrElse(PredictedShareScoreFeature, None), + dwellScore = candidate.features.getOrElse(PredictedDwellScoreFeature, None), + negativeFeedbackV2Score = + candidate.features.getOrElse(PredictedNegativeFeedbackV2ScoreFeature, None) + ) hmt.ScoredTweet( tweetId = candidate.candidateIdLong, authorId = candidate.features.get(AuthorIdFeature).get, // Cache the model score instead of the final score because rescoring is per-request score = candidate.features.getOrElse(WeightedModelScoreFeature, None), - suggestType = candidate.features.getOrElse(SuggestTypeFeature, None), + servedType = candidate.features.get(ServedTypeFeature), sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None), quotedTweetId = candidate.features.getOrElse(QuotedTweetIdFeature, None), @@ -78,6 +113,7 @@ class CachedScoredTweetsSideEffect @Inject() ( directedAtUserId = candidate.features.getOrElse(DirectedAtUserIdFeature, None), inNetwork = Some(candidate.features.getOrElse(InNetworkFeature, true)), sgsValidLikedByUserIds = Some(sgsValidLikedByUserIds), + validLikedByUserIds = Some(validLikedByUserIds), sgsValidFollowedByUserIds = Some(sgsValidFollowedByUserIds), topicId = candidate.features.getOrElse(TopicIdSocialContextFeature, None), topicFunctionalityType = candidate.features @@ -85,7 +121,6 @@ class CachedScoredTweetsSideEffect @Inject() ( TopicContextFunctionalityTypeMarshaller(_)), ancestors = if (ancestors.nonEmpty) Some(ancestors) else None, isReadFromCache = Some(true), - streamToKafka = Some(false), exclusiveConversationAuthorId = candidate.features .getOrElse(ExclusiveConversationAuthorIdFeature, None), authorMetadata = Some( @@ -94,14 +129,46 @@ class CachedScoredTweetsSideEffect @Inject() ( goldVerified = candidate.features.getOrElse(AuthorIsGoldVerifiedFeature, false), grayVerified = candidate.features.getOrElse(AuthorIsGrayVerifiedFeature, false), legacyVerified = candidate.features.getOrElse(AuthorIsLegacyVerifiedFeature, false), - creator = candidate.features.getOrElse(AuthorIsCreatorFeature, false) + creator = candidate.features.getOrElse(AuthorIsCreatorFeature, false), + followers = candidate.features.getOrElse(AuthorFollowersFeature, None) )), lastScoredTimestampMs = candidate.features .getOrElse(LastScoredTimestampMsFeature, Some(query.queryTime.inMilliseconds)), candidatePipelineIdentifier = candidate.features .getOrElse(CachedCandidatePipelineIdentifierFeature, Some(candidate.source.name)), tweetUrls = Some(candidate.features.getOrElse(TweetUrlsFeature, Seq.empty)), - perspectiveFilteredLikedByUserIds = Some(perspectiveFilteredLikedByUserIds) + perspectiveFilteredLikedByUserIds = None, + predictionRequestId = candidate.features.getOrElse(PredictionRequestIdFeature, None), + communityId = candidate.features.getOrElse(CommunityIdFeature, None), + communityName = candidate.features.getOrElse(CommunityNameFeature, None), + listId = candidate.features.getOrElse(ListIdFeature, None), + listName = candidate.features.getOrElse(ListNameFeature, None), + tweetTypeMetrics = candidate.features.getOrElse(TweetTypeMetricsFeature, None), + debugString = candidate.features.getOrElse(DebugStringFeature, None), + viralContentCreator = Some(candidate.features.getOrElse(ViralContentCreatorFeature, false)), + locationId = candidate.features.getOrElse(LocationIdFeature, None), + isArticle = Some(candidate.features.getOrElse(IsArticleFeature, false)), + hasVideo = Some(candidate.features.getOrElse(HasVideoFeature, false)), + videoDurationMs = candidate.features.getOrElse(VideoDurationMsFeature, None), + mediaIds = if (mediaIds.nonEmpty) Some(mediaIds) else None, + grokAnnotations = candidate.features.getOrElse(GrokAnnotationsFeature, None), + predictedScores = Some(predictedScores), + tweetMixerScore = candidate.features.getOrElse(TweetMixerScoreFeature, None), + clipClusterIdsFeature = Some( + hmt.ClipClusterIdsFeature( + tweetMediaClusterIdsFeature = Some( + candidate.features.getOrElse(TweetMediaClusterIdsFeature, Map.empty[Long, Long])), + clipImageClusterIdsFeature = + Some(candidate.features.getOrElse(ClipImageClusterIdsFeature, Map.empty[Long, Long])) + )), + grokSlopScoreFeature = candidate.features.getOrElse(GrokSlopScoreFeature, None), + mediaCompletionRate = candidate.features.getOrElse(TweetMediaCompletionRateFeature, None), + tweetText = candidate.features.getOrElse(TweetTextFeature, None), + sourceSignal = sourceSignal, + grokContentCreator = Some(candidate.features.getOrElse(GrokContentCreatorFeature, false)), + gorkContentCreator = Some(candidate.features.getOrElse(GorkContentCreatorFeature, false)), + // MultiModalEmbeddingsFeature are not cached because the value size is too large for memcache + multiModalEmbedding = None ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CommonFeaturesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CommonFeaturesSideEffect.scala new file mode 100644 index 000000000..ab766c3e4 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CommonFeaturesSideEffect.scala @@ -0,0 +1,96 @@ +package com.twitter.home_mixer.product.scored_tweets.side_effect + +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finatra.kafka.serde.ScalaSerdes +import com.twitter.home_mixer.functional_component.side_effect.CommonFeaturesPldrConverter +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeFeaturesFlag +import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeEventPublisher +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.inject.annotations.Flag +import com.twitter.logpipeline.client.common.EventPublisher +import com.twitter.product_mixer.component_library.side_effect.KafkaAndScribePublishingSideEffect +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.HasMarshalling +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.ml.kafka.serde.TBaseSerde +import com.twitter.timelines.suggests.common.poly_data_record.thriftjava.PolyDataRecord +import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.serialization.Serializer + +/** + * Publish common features sent to prediction service + some other features as PLDR format + * into both scribe logs and kafka + */ +@Singleton +class CommonFeaturesSideEffect @Inject() ( + serviceIdentifier: ServiceIdentifier, + commonFeaturesPldrConverter: CommonFeaturesPldrConverter, + @Flag(ScribeFeaturesFlag) enableScribeFeatures: Boolean, + @Named(CommonFeaturesScribeEventPublisher) override val logPipelinePublisher: EventPublisher[ + pldr.PolyDataRecord + ]) extends KafkaAndScribePublishingSideEffect[ + Long, + pldr.PolyDataRecord, + PipelineQuery, + HasMarshalling + ] + with Conditionally[PipelineQuery, HasMarshalling] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("CommonFeaturesSideEffect") + + private val kafkaTopic: String = serviceIdentifier.environment.toLowerCase match { + case "prod" => "tq_ct_common_features" + case _ => "tq_ct_common_features_staging" + } + + override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" + override val keySerde: Serializer[Long] = ScalaSerdes.Long.serializer() + override val valueSerde: Serializer[PolyDataRecord] = + TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer + override val clientId: String = "home_mixer_common_features_producer" + + override def enableScribePublishing(query: PipelineQuery): Boolean = enableScribeFeatures + + /** @see [[common.Conditionally.onlyIf]] */ + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Boolean = selectedCandidates.nonEmpty + + /** + * Build the record to be published to Kafka from query, selections and response + * + * @param query PipelineQuery + * @param selectedCandidates Result after Selectors are executed + * @param remainingCandidates Candidates which were not selected + * @param droppedCandidates Candidates dropped during selection + * @param response Result after Unmarshalling + * + * @return A sequence of to-be-published ProducerRecords + */ + override def buildRecords( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Seq[ProducerRecord[Long, PolyDataRecord]] = { + commonFeaturesPldrConverter + .getCommonFeaturesPldr(query, selectedCandidates).map { + case (predictionRequestId, pldr) => + new ProducerRecord(kafkaTopic, predictionRequestId, pldr) + }.toSeq + } + + override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5)) + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredCandidateFeatureKeysKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredCandidateFeatureKeysKafkaSideEffect.scala new file mode 100644 index 000000000..4412a56cd --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredCandidateFeatureKeysKafkaSideEffect.scala @@ -0,0 +1,257 @@ +package com.twitter.home_mixer.product.scored_tweets.side_effect + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.mysql.Client +import com.twitter.finagle.mysql.Transactions +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.util.DefaultTimer +import com.twitter.finatra.kafka.serde.ScalaSerdes +import com.twitter.home_mixer.functional_component.scorer.CandidateFeaturesDataRecordFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature +import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature +import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature +import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatureSet +import com.twitter.home_mixer.param.HomeGlobalParams.IsSelectedByHeavyRankerCountParam +import com.twitter.home_mixer.param.HomeMixerFlagName.DataRecordMetadataStoreConfigsYmlFlag +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableScoredCandidateFeatureKeysKafkaPublishingParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ScribedScoredCandidateNumParam +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.inject.annotations.Flag +import com.twitter.ml.api.DataRecordMerger +import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect +import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter +import com.twitter.product_mixer.core.feature.featuremap.datarecord.SpecificFeatures +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.HasMarshalling +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.CandidateAndCommonFeaturesStreamingUtils +import com.twitter.timelines.data_processing.jobs.light_ranking.light_ranking_features_prep.RecapPartialFeaturesForTwoTowerModels +import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.ScoredCandidateFeatureKeysAdapter +import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.ScoredCandidateFeatureKeysFields +import com.twitter.timelines.ml.kafka.serde.TBaseSerde +import com.twitter.timelines.ml.pldr.client.MysqlClientUtils +import com.twitter.timelines.ml.pldr.client.VersionedMetadataCacheClient +import com.twitter.timelines.ml.pldr.conversion.VersionIdAndFeatures +import com.twitter.timelines.suggests.common.data_record_metadata.{thriftscala => drmd} +import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} +import com.twitter.util.Try +import com.twitter.util.logging.Logging +import javax.inject.Inject +import javax.inject.Singleton +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.serialization.Serializer +import scala.collection.JavaConverters._ + +/** + * Pipeline side-effect that publishes scored candidate feature keys to a Kafka topic. + */ +@Singleton +class ScoredCandidateFeatureKeysKafkaSideEffect @Inject() ( + serviceIdentifier: ServiceIdentifier, + @Flag(DataRecordMetadataStoreConfigsYmlFlag) dataRecordMetadataStoreConfigsYml: String, + statsReceiver: StatsReceiver) + extends KafkaPublishingSideEffect[ + Long, + pldr.PolyDataRecord, + PipelineQuery, + HasMarshalling + ] + with Conditionally[PipelineQuery, HasMarshalling] + with Logging { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier( + "ScoredCandidateFeatureKeysKafka") + private val statScope: String = this.getClass.getSimpleName + + private val scopedStatsReceiver = statsReceiver.scope(statScope) + private val metadataFetchFailedCounter = scopedStatsReceiver.counter("metadataFetchFailed") + private val randomSelectedCandidatesStat = + scopedStatsReceiver.stat("randomSelectedCandidates") + private val randomDroppedCandidatesStat = + scopedStatsReceiver.stat("randomDroppedCandidates") + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Boolean = { + if (query.params.getBoolean(EnableScoredCandidateFeatureKeysKafkaPublishingParam)) { + val scribedCandidateNumber = query.params(ScribedScoredCandidateNumParam) + filterRankedCandidates(selectedCandidates).size >= scribedCandidateNumber && + filterRankedCandidates(droppedCandidates).size >= scribedCandidateNumber && selectedCandidates + .forall(!_.features.getOrElse(IsReadFromCacheFeature, false)) && remainingCandidates + .forall(!_.features.getOrElse(IsReadFromCacheFeature, false)) && droppedCandidates + .forall(!_.features.getOrElse(IsReadFromCacheFeature, false)) + } else false + } + + private val kafkaTopic: String = serviceIdentifier.environment.toLowerCase match { + case "prod" => "home_mixer_dropped_candidates_features" + case _ => "deep_retrieval_candidates_data_staging" + } + + override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" + override val keySerde: Serializer[Long] = ScalaSerdes.Long.serializer() + override val valueSerde: Serializer[pldr.PolyDataRecord] = + TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer + override val clientId: String = "home_mixer_dropped_candidate_feature_keys_producer" + + lazy private val dataRecordMetadataStoreClient: Option[Client with Transactions] = Try { + try { + val c = MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml) + logger.info(s"pldr mysql config: ${c.host} ${c.port} ${c.user} ${c.database}") + } catch { + case e: Throwable => + logger.error("pldr mysql error: " + e.toString) + } + + MysqlClientUtils.mysqlClientProvider( + MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml) + ) + }.toOption + + lazy private val versionedMetadataCacheClientOpt: Option[ + VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]] + ] = dataRecordMetadataStoreClient.map { mysqlClient => + new VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]]( + maximumSize = 1, + expireDurationOpt = None, + mysqlClient = mysqlClient, + transform = CandidateAndCommonFeaturesStreamingUtils.metadataTransformer, + statsReceiver = statsReceiver + ) + } + + versionedMetadataCacheClientOpt.foreach { + _.metadataFetchTimerTask( + CandidateAndCommonFeaturesStreamingUtils.metadataFetchKey, + metadataFetchTimer = DefaultTimer, + metadataFetchInterval = 90.seconds, + metadataFetchFailedCounter = metadataFetchFailedCounter + ) + } + + private val drMerger = new DataRecordMerger + private val predictedScoreFeaturesDataRecordAdapter = + new DataRecordConverter(SpecificFeatures(PredictedScoreFeatureSet)) + + override def buildRecords( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Seq[ProducerRecord[Long, pldr.PolyDataRecord]] = { + val rankedSelected = filterRankedCandidates(selectedCandidates) + .filterNot { candidate => + val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) + val isNsfw = annotations.flatMap(_.metadata.map(_.isNsfw)).getOrElse(false) + val isSoftNsfw = annotations.flatMap(_.metadata.map(_.isSoftNsfw)).getOrElse(false) + val isGore = annotations.flatMap(_.metadata.map(_.isGore)).getOrElse(false) + val isViolent = annotations.flatMap(_.metadata.map(_.isViolent)).getOrElse(false) + val isSpam = annotations.flatMap(_.metadata.map(_.isSpam)).getOrElse(false) + isNsfw || isSoftNsfw || isGore || isViolent || isSpam + } + val rankedDropped = filterRankedCandidates(droppedCandidates) + + val candidates = rankedSelected ++ rankedDropped + val isSelectedCandidateIds: Set[Long] = selectedCandidates.map(_.candidateIdLong).toSet + val candidatesHeavyRankerScoreBasedRank: Map[Long, Int] = candidates + .sortBy( + -_.features + .getOrElse(WeightedModelScoreFeature, None).getOrElse(Double.NegativeInfinity)).map( + _.candidateIdLong).zipWithIndex.toMap + val isSelectedByHeavyRankerCount = query.params(IsSelectedByHeavyRankerCountParam) + + val predictionRequestId = candidates.headOption.flatMap { candidate => + candidate.features.getOrElse(PredictionRequestIdFeature, None) + } + + val scribedCandidateNumber = query.params(ScribedScoredCandidateNumParam) + + val randomRankedSelected = selectRandomCandidates(rankedSelected, scribedCandidateNumber) + val randomRankedDropped = selectRandomCandidates(rankedDropped, scribedCandidateNumber) + + randomSelectedCandidatesStat.add(randomRankedSelected.size) + randomDroppedCandidatesStat.add(randomRankedDropped.size) + + val randomCandidates = randomRankedSelected ++ randomRankedDropped + + randomCandidates.flatMap { candidate => + val key = candidate.candidateIdLong + try { + val scoredCandidateFeatureKeysDataRecord = ScoredCandidateFeatureKeysAdapter + .adaptToDataRecords( + ScoredCandidateFeatureKeysFields( + viewerId = query.getRequiredUserId, + tweetId = key, + isSelected = isSelectedCandidateIds.contains(candidate.candidateIdLong), + isSelectedByHeavyRanker = candidatesHeavyRankerScoreBasedRank + .getOrElse( + candidate.candidateIdLong, + candidates.size) < isSelectedByHeavyRankerCount, + predictionRequestId = predictionRequestId, + requestTimeMs = Some(query.queryTime.inMilliseconds), + scribedCandidateNumber = scribedCandidateNumber, + viewerFollowsOriginalAuthor = + Some(candidate.features.getOrElse(InNetworkFeature, false)), + rankByHeavyRanker = + Some(candidatesHeavyRankerScoreBasedRank.getOrElse(key, candidates.size)) + ) + ).asScala.head + + val candidateFeaturesDataRecord = CandidateAndCommonFeaturesStreamingUtils + .extractFeaturesFromDrWithContext( + candidate.features.get(CandidateFeaturesDataRecordFeature), + RecapPartialFeaturesForTwoTowerModels.scoredCandidateFeatureContext + ) + drMerger.merge(scoredCandidateFeatureKeysDataRecord, candidateFeaturesDataRecord) + + val predictedScoreFeaturesDataRecord = + predictedScoreFeaturesDataRecordAdapter.toDataRecord(candidate.features) + drMerger.merge(scoredCandidateFeatureKeysDataRecord, predictedScoreFeaturesDataRecord) + + val convertedPlDrOpt = + CandidateAndCommonFeaturesStreamingUtils.candidateFeaturesToPolyDataRecord( + versionedMetadataCacheClientOpt = versionedMetadataCacheClientOpt, + candidateFeatures = scoredCandidateFeatureKeysDataRecord, + valueFormat = pldr.PolyDataRecord._Fields.LITE_COMPACT_DATA_RECORD + ) + + convertedPlDrOpt.map { convertedPlDr => + new ProducerRecord(kafkaTopic, key, convertedPlDr) + } + } catch { + case e: Exception => + logger.error( + s"Error while converting features to PolyDataRecord for candidateId: $key", + e + ) + None + } + } + } + + private def filterRankedCandidates( + candidates: Seq[CandidateWithDetails] + ): Seq[CandidateWithDetails] = + candidates.filter(_.features.contains(CandidateFeaturesDataRecordFeature)) + + private def selectRandomCandidates( + candidates: Seq[CandidateWithDetails], + count: Int + ): Seq[CandidateWithDetails] = + scala.util.Random.shuffle(CandidatesUtil.getItemCandidates(candidates)).take(count) + + override val alerts = Seq( + HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5) + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect.scala new file mode 100644 index 000000000..f0be58006 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect.scala @@ -0,0 +1,122 @@ +package com.twitter.home_mixer.product.scored_tweets.side_effect + +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finatra.kafka.serde.ScalaSerdes +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationScoreScribingParam +import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.HasMarshalling +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.strato.columns.content_understanding.content_understanding.thriftscala.ContentExplorationServingResult +import com.twitter.strato.columns.content_understanding.content_understanding.thriftscala.ContentExplorationServingResultsAnalysis +import com.twitter.util.logging.Logging +import javax.inject.Inject +import javax.inject.Singleton +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.serialization.Serializer + +/** + * Pipeline side-effect that publishes scores feature keys to a Kafka topic. + */ +@Singleton +class ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect @Inject() ( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver) + extends KafkaPublishingSideEffect[ + Long, + ContentExplorationServingResultsAnalysis, + PipelineQuery, + HasMarshalling + ] + with Conditionally[PipelineQuery, HasMarshalling] + with Logging { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier( + "ScoredContentExplorationCandidateScoreFeatureKafka") + private val statScope: String = this.getClass.getSimpleName + private val scopedStatsReceiver = statsReceiver.scope(statScope) + private val failedScribingCount = scopedStatsReceiver.scope("failed").counter() + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Boolean = + query.params(EnableContentExplorationScoreScribingParam) + + private val eligibleServedType: Set[hmt.ServedType] = Set( + hmt.ServedType.ForYouContentExploration, + hmt.ServedType.ForYouContentExplorationTier2, + hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i, + hmt.ServedType.ForYouContentExplorationTier2DeepRetrievalI2i, + hmt.ServedType.ForYouUserInterestSummary + ) + + private val kafkaTopic: String = serviceIdentifier.environment.toLowerCase match { + case "prod" => "content_exploration_ranking_analysis" + case _ => "content_exploration_ranking_analysis_devel" + } + + override val bootstrapServer: String = "" + override val keySerde: Serializer[Long] = ScalaSerdes.Long.serializer() + override val valueSerde: Serializer[ContentExplorationServingResultsAnalysis] = + ScalaSerdes.Thrift[ContentExplorationServingResultsAnalysis].serializer + override val clientId: String = "home_mixer_dropped_candidate_feature_keys_producer" + + override def buildRecords( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Seq[ProducerRecord[Long, ContentExplorationServingResultsAnalysis]] = { + try { + val userId: Long = query.getUserOrGuestId.getOrElse(-1L) + + val allCandidates = selectedCandidates ++ remainingCandidates ++ droppedCandidates + val eligible = allCandidates.filter(selectEligibleCandidates) + + val scoredResults: Seq[ContentExplorationServingResult] = eligible.map { cdd => + ContentExplorationServingResult( + cdd.candidateIdLong, + cdd.features.get(ScoreFeature), + cdd.features.get(WeightedModelScoreFeature), + Some(cdd.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined).toString) + ) + } + val analysisResult = ContentExplorationServingResultsAnalysis( + userId, + scoredResults + ) + Seq( + new ProducerRecord[Long, ContentExplorationServingResultsAnalysis]( + kafkaTopic, + userId, + analysisResult + ) + ) + } catch { + case _: Throwable => + failedScribingCount.incr() + Seq.empty + } + } + + def selectEligibleCandidates( + candidate: CandidateWithDetails + ): Boolean = { + val servedType = + candidate.features + .getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) + eligibleServedType.contains(servedType) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredPhoenixCandidatesKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredPhoenixCandidatesKafkaSideEffect.scala new file mode 100644 index 000000000..2e269f0d7 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredPhoenixCandidatesKafkaSideEffect.scala @@ -0,0 +1,188 @@ +package com.twitter.home_mixer.product.scored_tweets.side_effect + +import com.google.inject.name.Named +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finatra.kafka.serde.ScalaSerdes +import com.twitter.finatra.kafka.serde.UnKeyed +import com.twitter.finatra.kafka.serde.UnKeyedSerde +import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature.PhoenixPredictedScoreFeatures +import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatures +import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableScoredPhoenixCandidatesKafkaSideEffectParam +import com.twitter.home_mixer.util.PhoenixUtils.createCandidateSets +import com.twitter.home_mixer.util.PhoenixUtils.getPredictionResponseMap +import com.twitter.home_mixer.util.PhoenixUtils.getTweetInfoFromCandidates +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.HasMarshalling +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.util.MemoizingStatsReceiver +import com.twitter.stitch.Stitch +import com.twitter.timelines.timeline_logging.thriftscala.PredictionScore +import com.twitter.timelines.timeline_logging.thriftscala.ScoredCandidate +import com.twitter.util.logging.Logging +import com.x.user_action_sequence.ActionName +import com.x.user_action_sequence.PredictNextActionsRequest +import io.grpc.ManagedChannel +import javax.inject.Inject +import javax.inject.Singleton +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.serialization.Serializer + +/** + * Pipeline side-effect that publishes scored phoenix candidates to a Kafka topic. + */ +@Singleton +class ScoredPhoenixCandidatesKafkaSideEffect @Inject() ( + @Named("PhoenixClient") channelsMap: Map[PhoenixCluster.Value, Seq[ManagedChannel]], + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver) + extends KafkaPublishingSideEffect[ + UnKeyed, + ScoredCandidate, + PipelineQuery, + HasMarshalling + ] + with Conditionally[PipelineQuery, HasMarshalling] + with Logging { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier( + "ScoredPhoenixCandidatesKafka") + private val statScope: String = this.getClass.getSimpleName + + val memoizingStatsReceiver: MemoizingStatsReceiver = new MemoizingStatsReceiver( + statsReceiver.scope(statScope)) + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Boolean = { + query.params(EnableScoredPhoenixCandidatesKafkaSideEffectParam) && + query.features.flatMap(_.getOrElse(UserActionsFeature, None)).isDefined + } + + private val kafkaTopic: String = serviceIdentifier.environment.toLowerCase match { + case "prod" => "home_mixer_phoenix_scored_candidates" + case _ => "home_mixer_phoenix_scored_candidates_staging" + } + + override val bootstrapServer: String = "" + override val keySerde: Serializer[UnKeyed] = UnKeyedSerde.serializer() + override val valueSerde: Serializer[ScoredCandidate] = + ScalaSerdes.Thrift[ScoredCandidate].serializer + override val clientId: String = "home_mixer_phoenix_scored_candidate_producer" + + // Returns Map from phoenixCluster -> Map [Tweets -> Map [Actions -> Score]] + private def getPredictionResponsesAllClusters( + request: PredictNextActionsRequest + ): Stitch[Map[String, Map[Long, Map[ActionName, Double]]]] = { + val phoenixClusters = channelsMap.keySet.toSeq + val timeoutMs = 1000 + Stitch + .traverse(phoenixClusters) { phoenixCluster => + val channels = channelsMap(phoenixCluster) + getPredictionResponseMap( + request, + channels, + phoenixCluster.toString, + timeoutMs, + memoizingStatsReceiver) + .handle { + case _ => Map.empty[Long, Map[ActionName, Double]] + } + .map(phoenixCluster.toString -> _) + }.map(_.toMap) + } + + // Not defined buildRecords as we override apply to use buildRecordsStitch + override def buildRecords( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Seq[ProducerRecord[UnKeyed, ScoredCandidate]] = { + ??? + } + + def buildRecordsStitch( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + ): Stitch[Seq[ProducerRecord[UnKeyed, ScoredCandidate]]] = { + val nonCachedSelectedCandidates = + selectedCandidates.filterNot(_.features.getOrElse(IsReadFromCacheFeature, false)) + + val candidates = nonCachedSelectedCandidates.map(_.getCandidate[TweetCandidate]()) + val featureMaps = nonCachedSelectedCandidates.map(_.features) + val tweetInfos = getTweetInfoFromCandidates(candidates, featureMaps) + val request = createCandidateSets(query, tweetInfos) + val predictionsMapStitch = + getPredictionResponsesAllClusters(request) + + val scoredCandidatesStitch = Stitch.traverse(nonCachedSelectedCandidates) { candidate => + val tweetId = candidate.candidateIdLong + val authorId = candidate.features.getOrElse(AuthorIdFeature, None) + val sourceTweetId = + candidate.features.getOrElse(SourceTweetIdFeature, None) match { + case Some(sourceTweetId) => sourceTweetId + case _ => tweetId + } + predictionsMapStitch.map { predictionsMap => + val phoenixPredictionScores = predictionsMap.flatMap { + case (phoenixCluster, scoresMapPerTweet) => + val actionPredictionsMap = scoresMapPerTweet.getOrElse(sourceTweetId, Map.empty) + PhoenixPredictedScoreFeatures.map { feature => + val predictions = feature.actions.flatMap(actionPredictionsMap.get) + val score = if (predictions.nonEmpty) Some(predictions.max) else None + val featureString = f"phoenix.$phoenixCluster.${feature.featureName}" + PredictionScore(Some(featureString), score) + } + }.toSeq + val prodScores = PredictedScoreFeatures.map { feature => + PredictionScore(Some(feature.statName), candidate.features.getOrElse(feature, None)) + } + ScoredCandidate( + tweetId = tweetId, + authorId = authorId, + viewerId = query.getOptionalUserId, + sourceTweetId = Some(sourceTweetId), + servedRequestId = query.features.flatMap(_.getOrElse(PredictionRequestIdFeature, None)), + requestTimeMs = Some(query.queryTime.inMillis), + predictionScores = Some((phoenixPredictionScores ++ prodScores).toSet) + ) + } + } + + scoredCandidatesStitch.map { predictionScoresSeq => + predictionScoresSeq.map { + new ProducerRecord(kafkaTopic, new UnKeyed, _) + } + } + } + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, HasMarshalling] + ): Stitch[Unit] = { + val recordsStitch = buildRecordsStitch( + query = inputs.query, + selectedCandidates = inputs.selectedCandidates, + ) + + recordsStitch.flatMap { records => + Stitch + .traverse(records) { record => + Stitch.callFuture(kafkaProducer.send(record)) + } + }.unit + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredStatsSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredStatsSideEffect.scala new file mode 100644 index 000000000..6a5f3857a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredStatsSideEffect.scala @@ -0,0 +1,562 @@ +package com.twitter.home_mixer.product.scored_tweets.side_effect + +import com.twitter.core_workflows.user_model.thriftscala.UserState +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.NullStatsReceiver.NullCounter +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.LowSignalUserFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedAuthorIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.SlopAuthorScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature +import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature +import com.twitter.home_mixer.model.PredictedScoreFeature +import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatures +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.AuthorListForDataCollectionParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.RequestNormalizedScoresParam +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsTweetMixerCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweets +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines +import com.twitter.product_mixer.core.model.common.presentation.CandidateSourcePosition +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.util.Memoize +import com.twitter.util.logging.Logging +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class ScoredStatsSideEffect @Inject() ( + statsReceiver: StatsReceiver) + extends PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] + with Logging { + + import ScoredStatsSideEffect._ + override val identifier: SideEffectIdentifier = SideEffectIdentifier("ScoredStats") + + private val baseStatsReceiver = statsReceiver.scope(identifier.toString) + private val authorStatsReceiver = baseStatsReceiver.scope("Author") + private val authorScoreStatsReceiver = baseStatsReceiver.scope("AuthorScore10000x") + private val servedTypeStatsReceiver = baseStatsReceiver.scope("ServedType") + private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance") + private val grokAnnotationsStatsReceiver = baseStatsReceiver.scope("GrokAnnotations") + + private val inNetworkStatsReceiver = contentBalanceStatsReceiver.scope("InNetwork") + private val outOfNetworkStatsReceiver = contentBalanceStatsReceiver.scope("OutOfNetwork") + private val replyStatsReceiver = contentBalanceStatsReceiver.scope("Reply") + private val originalStatsReceiver = contentBalanceStatsReceiver.scope("Original") + private val retweetStatsReceiver = contentBalanceStatsReceiver.scope("Retweet") + + private val tweetMixerRecallStatsReceiver = baseStatsReceiver.scope("TweetMixerRecall10000x") + private val deepRetrievalRecallStatsReceiver = + baseStatsReceiver.scope("DeepRetrievalRecall10000x") + private val tweetMixerTweetsLandOnTopKStatsReceiver = baseStatsReceiver.scope("TweetMixerTweets") + private val deepRetrievalTweetsLandOnTopKStatsReceiver = + baseStatsReceiver.scope("DeepRetrievalTweets") + + private val grokAnnotationsInStatsReceiver = grokAnnotationsStatsReceiver.scope("InNetwork") + private val grokAnnotationsOonStatsReceiver = grokAnnotationsStatsReceiver.scope("OutOfNetwork") + private val grokAnnotationsReplyStatsReceiver = grokAnnotationsStatsReceiver.scope("Reply") + + private val signalStatsReceiver = baseStatsReceiver.scope("signal") + private val authorDiversityStatsReceiver = baseStatsReceiver.scope("authorDiversity") + private val impressedAuthorSizeStat = authorDiversityStatsReceiver.stat("impressedAuthorSize") + private val meanImpressedAuthorCountStat = + authorDiversityStatsReceiver.stat("meanImpressedAuthorCount1000x") + + private val grokSlopScoreStatsReceiver = baseStatsReceiver.scope("GrokSlopScore") + + private val inNetAuthorStatsReceiver = baseStatsReceiver.scope("InNetAuthor") + + private val StatsReadabilityMultiplier = 1000 + private val SmallFollowGraphSize = 25 + + private val StatsEligibleGrokTopics = Set( + "News", + "Sports", + "Entertainment", + "Business & Finance", + "Technology", + "Politics", + "Fashion & Beauty", + "Gaming", + "Lifestyle", + "Movies & TV", + "Music", + "Travel", + "Food", + "Comedy", + "Health & Fitness" + ) + + // Group of stats to collect per model + case class ModelStats(normalized: Boolean, rankingMode: String) { + val normalizedScope = if (normalized) "normalized" else "weighted" + val scopedStatsReceiver = baseStatsReceiver.scope(normalizedScope).scope(rankingMode) + val PredictedScoreCounterName = f"predictedScore${StatsReadabilityMultiplier}xSum" + val requestCounter = scopedStatsReceiver.counter("candidatesSum") + private val predictedScoreCounters: Map[PredictedScoreFeature, Counter] = + PredictedScoreFeatures.map { scoreFeature => + ( + scoreFeature, + scopedStatsReceiver.counter(scoreFeature.statName, PredictedScoreCounterName)) + }.toMap + + def getPredictedScoreCounter(scoreFeature: PredictedScoreFeature): Counter = + predictedScoreCounters.getOrElse(scoreFeature, NullCounter) + } + + // Memoize stats object so they are not created per request + private val statsPerModel = Memoize[(Boolean, String), ModelStats] { + case (normalized, rankingMode) => ModelStats(normalized, rankingMode) + } + + override def apply( + inputs: PipelineResultSideEffect.Inputs[ScoredTweetsQuery, ScoredTweetsResponse] + ): Stitch[Unit] = { + val scopeCandidateMap = Map( + "selected" -> inputs.selectedCandidates, + "dropped" -> inputs.droppedCandidates, + "remaining" -> inputs.remainingCandidates + ) + // We choose 50 because we don't have per head scores for cached candidates + val topSelectedCandidates = inputs.selectedCandidates.take(50) + + val impressedTweetIds = + inputs.query.features.map(_.getOrElse(ImpressedTweets, Seq.empty)).getOrElse(Seq.empty).toSet + val servedAuthorMap: Map[Long, Seq[Long]] = + inputs.query.features + .map(_.getOrElse(ServedAuthorIdsFeature, Map.empty[Long, Seq[Long]])) + .getOrElse(Map.empty[Long, Seq[Long]]) + recordAuthorDiversityStats(impressedTweetIds, servedAuthorMap) + + recordScoreStats( + topSelectedCandidates, + inputs.query.params(RequestNormalizedScoresParam), + inputs.query.params(ScoredTweetsParam.TweetMixerRankingModeForStatsRecallAtKParam), + inputs.query + ) + + val allCandidates: Seq[CandidateWithDetails] = + inputs.selectedCandidates ++ inputs.droppedCandidates + recordInNetworkAuthorStats(allCandidates, inputs.query, "retrieved") + recordInNetworkAuthorStats(inputs.selectedCandidates, inputs.query, "selected") + + scopeCandidateMap.map { + case (scope, candidates) => + recordAuthorStats(candidates, scope, inputs.query.params(AuthorListForDataCollectionParam)) + recordServedTypeStats(candidates, scope) + recordGrokAnnotationsStats(candidates, scope) + recordContentBalanceStats( + candidates, + scope, + inputs.query.params(ScoredTweetsParam.TweetMixerRankingModeForStatsRecallAtKParam) + ) + recordHasVideoStats(candidates, scope) + recordHasMediaStats(candidates, scope) + recordSlopScoreStats(candidates, scope) + } + + recordTweetMixerRecallTopKStats( + inputs.selectedCandidates.filter( + _.source != CachedScoredTweetsCandidatePipelineConfig.Identifier), + inputs.droppedCandidates.filter( + _.source != CachedScoredTweetsCandidatePipelineConfig.Identifier), + inputs.query.params(ScoredTweetsParam.TweetMixerRankingModeForStatsRecallAtKParam) + ) + + recordSignalStats(inputs.query) + + Stitch.Unit + } + + private def recordAuthorDiversityStats( + impressedTweetIds: Set[Long], + servedAuthorMap: Map[Long, Seq[Long]] + ): Unit = { + val authorFreq = servedAuthorMap + .collect { + case (authorId, tweetIds) => + val impressedCount = tweetIds.count(impressedTweetIds.contains) + if (impressedCount > 0) Some(authorId -> impressedCount) else None + }.flatten.toMap + + val impressedAuthorSize = authorFreq.size + val meanImpressedAuthorCount = + if (authorFreq.isEmpty) 0.0f else authorFreq.values.sum.toFloat / impressedAuthorSize + + impressedAuthorSizeStat.add(impressedAuthorSize) + meanImpressedAuthorCountStat.add(meanImpressedAuthorCount * 1000) + } + + private def recordScoreStats( + candidates: Seq[CandidateWithDetails], + normalized: Boolean, + rankingMode: String, + query: PipelineQuery + ): Unit = { + val modelStats = statsPerModel((normalized, rankingMode)) + modelStats.requestCounter.incr() + PredictedScoreFeatures.map { predictedScoreFeature => + val counter = modelStats.getPredictedScoreCounter(predictedScoreFeature) + val predictedScoreSum = candidates + .map { candidate => + predictedScoreFeature + .extractScore(candidate.features, query) + .getOrElse(0.0) * StatsReadabilityMultiplier + }.sum.toInt + counter.incr(predictedScoreSum) + } + } + + private def recordTweetMixerRecallTopKStats( + selectedCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + rankingMode: String + ): Unit = { + val allCandidates = selectedCandidates ++ droppedCandidates + val allCandidatesByHeavyRanker = allCandidates + .sortBy( + -_.features.getOrElse(WeightedModelScoreFeature, None).getOrElse(Double.NegativeInfinity)) + val tweetMixerTweets = allCandidatesByHeavyRanker.filter(fromTweetMixer) + val tweetMixerTweetsSortByRank = + tweetMixerTweets.sortBy(candidate => candidate.features.get(CandidateSourcePosition)) + + val deepRetrievalTweets = tweetMixerTweets.filter(tweetMixerTweet => + tweetMixerTweet.features.get( + ServedTypeFeature) == hmt.ServedType.ForYouDeepRetrieval || tweetMixerTweet.features.get( + ServedTypeFeature) == hmt.ServedType.ForYouEvergreenDeepRetrieval) + val deepRetrievalTweetsSortByRank = + deepRetrievalTweets.sortBy(_.features.get(CandidateSourcePosition)) + + if (deepRetrievalTweets.nonEmpty) { + HeavyRankerTopK.foreach { K => + deepRetrievalTweetsLandOnTopKStatsReceiver + .scope(rankingMode) + .stat(s"LandOnTop${K}") + .add(allCandidatesByHeavyRanker + .take(K).count { candidate => + fromServedType(candidate, hmt.ServedType.ForYouDeepRetrieval) || fromServedType( + candidate, + hmt.ServedType.ForYouEvergreenDeepRetrieval) + }) + } + + deepRetrievalTweetsLandOnTopKStatsReceiver + .scope(rankingMode) + .stat("LandInSelected") + .add(selectedCandidates.count { candidate => + fromServedType(candidate, hmt.ServedType.ForYouDeepRetrieval) || fromServedType( + candidate, + hmt.ServedType.ForYouEvergreenDeepRetrieval) + }) + + DeepRetrievalRecallTopK.foreach { K => + val rankedByHeavyRankerScoreTopK = deepRetrievalTweets.take(K) + val rankedByTweetMixerRankTopK = deepRetrievalTweetsSortByRank.take(K) + val intersectTweets = rankedByHeavyRankerScoreTopK + .map(candidate => candidate.candidateIdLong) + .intersect(rankedByTweetMixerRankTopK.map(candidate => candidate.candidateIdLong)) + + deepRetrievalRecallStatsReceiver + .scope(rankingMode) + .stat(s"RecallAt${K}") + .add(intersectTweets.size * 10000 / Math.min(K, deepRetrievalTweets.size)) + } + } + + if (tweetMixerTweets.nonEmpty) { + TweetMixerRecallTopK.foreach { K => + tweetMixerTweetsLandOnTopKStatsReceiver + .scope(rankingMode) + .stat(s"LandOnTop${K}") + .add(allCandidatesByHeavyRanker + .take(K).count(fromTweetMixer)) + } + + tweetMixerTweetsLandOnTopKStatsReceiver + .scope(rankingMode) + .stat("LandInSelected") + .add(selectedCandidates.count(fromTweetMixer)) + + TweetMixerRecallTopK.foreach { K => + val rankedByHeavyRankerScoreTopK = tweetMixerTweets.take(K) + val rankedByTweetMixerRankTopK = tweetMixerTweetsSortByRank.take(K) + val intersectTweets = rankedByHeavyRankerScoreTopK + .map(candidate => candidate.candidateIdLong) + .intersect(rankedByTweetMixerRankTopK.map(candidate => candidate.candidateIdLong)) + + tweetMixerRecallStatsReceiver + .scope(rankingMode) + .stat(s"RecallAt${K}") + .add(intersectTweets.size * 10000 / Math.min(K, tweetMixerTweets.size)) + } + } + } + + def recordAuthorStats( + candidates: Seq[CandidateWithDetails], + scope: String, + authors: Set[Long] + ): Unit = { + candidates + .filter { candidate => + candidate.features.getOrElse(AuthorIdFeature, None).exists(authors.contains) && + // Only include original tweets + (!candidate.features.getOrElse(IsRetweetFeature, false)) && + candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty + } + .groupBy { candidate => + val servedType = candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) + (servedType.name, candidate.features.get(AuthorIdFeature).get) + } + .foreach { + case ((servedType, authorId), authorCandidates) => + val authorStr = authorId.toString.takeRight(9) + authorStatsReceiver + .scope(scope).scope(authorStr).counter(servedType) + .incr(authorCandidates.size) + + authorCandidates.map { candidate => + val score = + candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0).toFloat * 10000 + authorScoreStatsReceiver.scope(scope).scope(authorStr).stat(servedType).add(score) + } + } + } + + def recordServedTypeStats( + candidates: Seq[CandidateWithDetails], + scope: String + ): Unit = { + candidates.groupBy(getServedType).foreach { + case (servedType, servedTypeCandidates) => + servedTypeStatsReceiver.scope(scope).counter(servedType).incr(servedTypeCandidates.size) + } + } + + private def recordHasVideoStats( + candidates: Seq[CandidateWithDetails], + scope: String + ): Unit = { + candidates.groupBy(_.features.getOrElse(HasVideoFeature, false)).foreach { + case (hasVideo, hasVideoCandidates) => + servedTypeStatsReceiver + .scope(scope).counter(if (hasVideo) "HasVideo" else "NoVideo").incr( + hasVideoCandidates.size) + } + } + + private def recordHasMediaStats( + candidates: Seq[CandidateWithDetails], + scope: String + ): Unit = { + candidates.groupBy(_.features.getOrElse(TweetMediaIdsFeature, Seq[Long]()).nonEmpty).foreach { + case (hasMedia, hasMediaCandidates) => + servedTypeStatsReceiver + .scope(scope).counter(if (hasMedia) "HasMedia" else "NoMedia").incr( + hasMediaCandidates.size) + } + } + + def recordGrokAnnotationsStats( + candidates: Seq[CandidateWithDetails], + scope: String + ): Unit = { + candidates.foreach { candidate => + val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) + + val topics = annotations + .map(_.topics.filter(StatsEligibleGrokTopics.contains(_))) + .filter(_.nonEmpty) + + val in = candidate.features.getOrElse(InNetworkFeature, true) + val reply = candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined + val baseStat = + if (reply) grokAnnotationsReplyStatsReceiver + else if (in) grokAnnotationsInStatsReceiver + else grokAnnotationsOonStatsReceiver + + if (topics.isDefined) baseStat.scope(scope).counter("Available").incr() + else baseStat.scope(scope).counter("Unavailable").incr() + + topics.map(_.map(topic => baseStat.counter(topic).incr())) + } + } + + def recordSignalStats( + query: ScoredTweetsQuery + ): Unit = { + val userState = query.features.flatMap(_.getOrElse(UserStateFeature, None)) + val userStateStr = userState + .map { + case UserState.New => "new" + case UserState.NearZero => "nearZero" + case UserState.VeryLight => "veryLight" + case UserState.Light => "light" + case UserState.MediumTweeter => "medium" + case UserState.MediumNonTweeter => "medium" + case UserState.HeavyNonTweeter => "heavy" + case UserState.HeavyTweeter => "heavy" + case UserState.EnumUnknownUserState(_) => "none" + }.getOrElse("none") + + val followGraphSize = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty).size) + val lowFollow = followGraphSize.exists(_ < SmallFollowGraphSize) + val lowSignalUser = + query.features.map(_.getOrElse(LowSignalUserFeature, false)).getOrElse(false) + + val stats = signalStatsReceiver.scope(userStateStr) + + if (lowSignalUser) { + if (lowFollow) stats.counter("lowSignal-lowFollow").incr() + else stats.counter("lowSignal-okFollow").incr() + } else { + if (lowFollow) stats.counter("okSignal-lowFollow").incr() + else stats.counter("okSignal-okFollow").incr() + } + } + + def recordContentBalanceStats( + candidates: Seq[CandidateWithDetails], + scope: String, + rankingMode: String + ): Unit = { + // Split by network type + val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true)) + inNetworkStatsReceiver.counter(scope).incr(in.size) + outOfNetworkStatsReceiver.counter(scope).incr(oon.size) + + // Track tweet types for all candidates (maintaining backward compatibility) + val (reply, nonReply) = + candidates.partition(_.features.getOrElse(InReplyToTweetIdFeature, None).isDefined) + replyStatsReceiver.counter(scope).incr(reply.size) + replyStatsReceiver.scope(rankingMode).counter(scope).incr(reply.size) + originalStatsReceiver.scope(rankingMode).counter(scope).incr(nonReply.size) + originalStatsReceiver.counter(scope).incr(nonReply.size) + + // Enhanced tracking: retweets, replies, and original posts for both in-network and out-of-network + val (retweets, nonRetweets) = + candidates.partition(_.features.getOrElse(IsRetweetFeature, false)) + val (replies, originalPosts) = + nonRetweets.partition(_.features.getOrElse(InReplyToTweetIdFeature, None).isDefined) + + // Record retweet stats + retweetStatsReceiver.counter(scope).incr(retweets.size) + retweetStatsReceiver.scope(rankingMode).counter(scope).incr(retweets.size) + + // Track by network type + val (inRetweets, oonRetweets) = retweets.partition(_.features.getOrElse(InNetworkFeature, true)) + val (inReplies, oonReplies) = replies.partition(_.features.getOrElse(InNetworkFeature, true)) + val (inOriginalPosts, oonOriginalPosts) = + originalPosts.partition(_.features.getOrElse(InNetworkFeature, true)) + + // In-network tweet type stats + retweetStatsReceiver.scope("InNetwork").counter(scope).incr(inRetweets.size) + retweetStatsReceiver.scope("InNetwork").scope(rankingMode).counter(scope).incr(inRetweets.size) + replyStatsReceiver.scope("InNetwork").counter(scope).incr(inReplies.size) + replyStatsReceiver.scope("InNetwork").scope(rankingMode).counter(scope).incr(inReplies.size) + originalStatsReceiver.scope("InNetwork").counter(scope).incr(inOriginalPosts.size) + originalStatsReceiver + .scope("InNetwork").scope(rankingMode).counter(scope).incr(inOriginalPosts.size) + + // Out-of-network tweet type stats + retweetStatsReceiver.scope("OutOfNetwork").counter(scope).incr(oonRetweets.size) + retweetStatsReceiver + .scope("OutOfNetwork").scope(rankingMode).counter(scope).incr(oonRetweets.size) + replyStatsReceiver.scope("OutOfNetwork").counter(scope).incr(oonReplies.size) + replyStatsReceiver.scope("OutOfNetwork").scope(rankingMode).counter(scope).incr(oonReplies.size) + originalStatsReceiver.scope("OutOfNetwork").counter(scope).incr(oonOriginalPosts.size) + originalStatsReceiver + .scope("OutOfNetwork").scope(rankingMode).counter(scope).incr(oonOriginalPosts.size) + } + + private def getServedType(candidate: CandidateWithDetails): String = + candidate.features.get(ServedTypeFeature).name + + private def fromTweetMixer(candidate: CandidateWithDetails): Boolean = { + candidate.features + .get(CandidatePipelines) + .contains(ScoredTweetsTweetMixerCandidatePipelineConfig.Identifier) + } + + private def fromServedType( + candidate: CandidateWithDetails, + servedType: hmt.ServedType + ): Boolean = candidate.features.get(ServedTypeFeature) == servedType + + private def recordSlopScoreStats( + candidates: Seq[CandidateWithDetails], + scope: String + ): Unit = { + val count1 = candidates.count(_.features.getOrElse(SlopAuthorScoreFeature, None).contains(1)) + val count2 = candidates.count(_.features.getOrElse(SlopAuthorScoreFeature, None).contains(2)) + val count3 = candidates.count(_.features.getOrElse(SlopAuthorScoreFeature, None).contains(3)) + val noneScore = candidates.count(_.features.getOrElse(SlopAuthorScoreFeature, None).isEmpty) + val total = candidates.size + grokSlopScoreStatsReceiver.scope(scope).counter("count1").incr(count1) + grokSlopScoreStatsReceiver.scope(scope).counter("count2").incr(count2) + grokSlopScoreStatsReceiver.scope(scope).counter("count3").incr(count3) + grokSlopScoreStatsReceiver.scope(scope).counter("none").incr(noneScore) + grokSlopScoreStatsReceiver.scope(scope).counter("total").incr(total) + } + + def recordInNetworkAuthorStats( + candidates: Seq[CandidateWithDetails], + query: ScoredTweetsQuery, + scope: String + ): Unit = { + val inNet = candidates.filter { candidate => + candidate.features.getOrElse(InNetworkFeature, true) && + (candidate.features.getOrElse(FromInNetworkSourceFeature, false)) + } + val total = inNet.size + if (total == 0) return + + val authorCounts: Seq[Int] = inNet + .flatMap(_.features.getOrElse(AuthorIdFeature, None)) + .groupBy(identity).map { case (k, v) => (k, v.size) }.values.toSeq + + val followGraphSize = + query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty).size).getOrElse(0) + val sr = inNetAuthorStatsReceiver.scope(scope) + if (followGraphSize > 0) { + val recall1000xSGS = (authorCounts.size * 1000) / followGraphSize + sr.stat("Recall1000xSGS").add(recall1000xSGS) + } + + val sorted = authorCounts.sorted(Ordering[Int].reverse) + def share(k: Int): Int = (sorted.take(k).sum * 1000) / total + + sr.stat("Top1Share1000x").add(share(1)) + sr.stat("Top3Share1000x").add(share(3)) + sr.stat("Top10Share1000x").add(share(10)) + } + + override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()) +} + +object ScoredStatsSideEffect { + val TweetMixerRecallTopK = Seq(25, 50, 100, 200, 400) + val DeepRetrievalRecallTopK = Seq(50, 100, 200) + val HeavyRankerTopK = Seq(1, 5, 10, 25, 50, 100, 200, 400) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredTweetsDiversityStatsSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredTweetsDiversityStatsSideEffect.scala new file mode 100644 index 000000000..399099aa0 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredTweetsDiversityStatsSideEffect.scala @@ -0,0 +1,81 @@ +package com.twitter.home_mixer.product.scored_tweets.side_effect + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.functional_component.feature_hydrator.SimClustersLogFavBasedTweetFeature +import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.simclusters_features.SimclustersFeaturesAdapter.SimclustersSparseTweetEmbeddingsFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.CategoryDiversityRescoringParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.CategoryDiversityKParam +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton +import com.twitter.ml.api.DataRecord +import scala.jdk.CollectionConverters._ + +@Singleton +case class ScoredTweetsDiversityStatsSideEffect @Inject() ( + statsReceiver: StatsReceiver) + extends PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] { + + import ScoredTweetsDiversityStatsSideEffect._ + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("ScoredTweetsDiversityStats") + private val baseStatsReceiver = statsReceiver.scope(identifier.toString) + private val diversityStatsReceiverMap = diversityTopK.map { k => + k.toString -> baseStatsReceiver.scope("numUniqCategories@k=" + k.toString) + }.toMap + + override def apply( + inputs: PipelineResultSideEffect.Inputs[ScoredTweetsQuery, ScoredTweetsResponse] + ): Stitch[Unit] = { + val query = inputs.query + val selectedCandidates = inputs.selectedCandidates + + if (query.params(CategoryDiversityRescoringParam)) { + diversityTopK.foreach { topK => + getNumUniqueCategoriesStats( + query, + selectedCandidates, + topK + ) + } + } + + Stitch.Unit + } + + private def getNumUniqueCategoriesStats( + query: ScoredTweetsQuery, + selectedCandidates: Seq[CandidateWithDetails], + topK: Int + ): Unit = { + val topKSelectedCandidates = selectedCandidates + .sortBy(_.features.getOrElse(ScoreFeature, None).getOrElse(0.0)) + .take(topK) + + val categories = topKSelectedCandidates.map { candidate => + val topKClustersMap = + Option( + candidate.features + .getOrElse(SimClustersLogFavBasedTweetFeature, new DataRecord()) + .getSparseContinuousFeatures + ).flatMap(mapOpt => + Option(mapOpt.get(SimclustersSparseTweetEmbeddingsFeature.getFeatureId))) + topKClustersMap + .map(_.asScala.toSeq.sortBy(-_._2).take(query.params(CategoryDiversityKParam)).map(_._1)) + .getOrElse(Seq.empty) + } + // assign the num of unique categories to the stats receiver + val uniqueCategories = categories.flatten.toSet + diversityStatsReceiverMap(topK.toString).stat("numUniqCategories").add(uniqueCategories.size) + } +} + +object ScoredTweetsDiversityStatsSideEffect { + private val diversityTopK = Array(5, 15, 30) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeScoredCandidatesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeScoredCandidatesSideEffect.scala index ef7e3b41a..f6a2531ad 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeScoredCandidatesSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeScoredCandidatesSideEffect.scala @@ -8,17 +8,30 @@ import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsGoreFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsLowQualityFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsNsfwFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsOcrFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsSpamFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokIsViolentFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokTopCategoryFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetMixerScoreFeature +import com.twitter.home_mixer.model.PredictedScoreFeature import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeScoredCandidatesFlag import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableScribeScoredCandidatesParam +import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect @@ -27,10 +40,12 @@ import com.twitter.product_mixer.core.functional_component.side_effect.PipelineR import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import com.twitter.timelines.timeline_logging.{thriftscala => t} +import com.twitter.util.logging.Logging + import javax.inject.Inject import javax.inject.Singleton -import com.twitter.util.logging.Logging /** * Side effect that logs scored candidates from scoring pipelines @@ -89,6 +104,14 @@ class ScribeScoredCandidatesSideEffect @Inject() ( query: ScoredTweetsQuery, isDropped: Boolean ): t.ScoredCandidate = { + val grokMetadata = t.GrokMetadata( + isGore = candidate.features.getOrElse(GrokIsGoreFeature, None), + isNsfw = candidate.features.getOrElse(GrokIsNsfwFeature, None), + isSpam = candidate.features.getOrElse(GrokIsSpamFeature, None), + isViolent = candidate.features.getOrElse(GrokIsViolentFeature, None), + isLowQuality = candidate.features.getOrElse(GrokIsLowQualityFeature, None), + isOcr = candidate.features.getOrElse(GrokIsOcrFeature, None) + ) t.ScoredCandidate( tweetId = candidate.candidateIdLong, viewerId = query.getOptionalUserId, @@ -96,7 +119,7 @@ class ScribeScoredCandidatesSideEffect @Inject() ( traceId = Some(Trace.id.traceId.toLong), requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)), score = candidate.features.getOrElse(ScoreFeature, None), - suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name), + suggestType = Some(candidate.features.get(ServedTypeFeature).name), isInNetwork = candidate.features.getTry(FromInNetworkSourceFeature).toOption, inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), inReplyToUserId = candidate.features.getOrElse(InReplyToUserIdFeature, None), @@ -110,10 +133,40 @@ class ScribeScoredCandidatesSideEffect @Inject() ( candidatePipelineIdentifier = candidate.features.getTry(CandidatePipelines).toOption.map(_.head.name), earlybirdScore = candidate.features.getOrElse(EarlybirdScoreFeature, None), - isDropped = Some(isDropped) + isDropped = Some(isDropped), + servedRequestId = query.features.flatMap(_.getOrElse(ServedIdFeature, None)), + sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), + predictionScores = + Some(extractPredictionScores(candidate) + extractCandidateSourceScore(candidate)), + grokAnnotation = Some( + t.GrokAnnotation( + category = candidate.features + .getOrElse(GrokTopCategoryFeature, None), + grokMetadata = Some(grokMetadata) + )), + sourceSignal = candidate.features.getOrElse(SourceSignalFeature, None).map { signal => + t.SourceSignal( + id = Some(signal.id), + signalType = signal.signalType, + signalEntity = signal.signalEntity, + authorId = signal.authorId, + ) + } ) } + private def extractPredictionScores(candidate: CandidateWithDetails): Set[t.PredictionScore] = + PredictedScoreFeature.PredictedScoreFeatureSet + .map(feature => + t.PredictionScore(Some(feature.featureName), candidate.features.getOrElse(feature, None))) + .filter(_.score.isDefined) + + private def extractCandidateSourceScore(candidate: CandidateWithDetails): t.PredictionScore = + t.PredictionScore( + Some(TimelinesSharedFeatures.CANDIDATE_SOURCE_SCORE.getFeatureName), + candidate.features + .getOrElse(TweetMixerScoreFeature, None)) + private def convertSeqFeature[T]( candidateWithDetails: CandidateWithDetails, feature: Feature[_, Seq[T]] @@ -123,4 +176,6 @@ class ScribeScoredCandidatesSideEffect @Inject() ( .getOrElse(feature, Seq.empty)).filter(_.nonEmpty) override val logPipelinePublisher: EventPublisher[t.ScoredCandidate] = eventBusPublisher + + override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util/BUILD.bazel new file mode 100644 index 000000000..0963e0465 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util/BUILD.bazel @@ -0,0 +1,12 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + dependencies = [ + "3rdparty/jvm/com/github/nscala_time", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/embedding", + "snowflake/src/main/scala/com/twitter/snowflake/id", + "src/thrift/com/twitter/timelines/control_ai:timeline-control-ai-thrift-scala", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util/ControlAiUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util/ControlAiUtil.scala new file mode 100644 index 000000000..e0870c281 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util/ControlAiUtil.scala @@ -0,0 +1,101 @@ +package com.twitter.home_mixer.product.scored_tweets.util + +import com.github.nscala_time.time.Imports.LocalDate +import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.embedding.TweetTextV8EmbeddingFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.timelines.control_ai.control.{thriftscala => ci} + +object ControlAiUtil { + private def isDotProductClose( + vector1: Option[Seq[Double]], + vector2: Option[Seq[Double]], + threshold: Double + ): Boolean = { + (vector1, vector2) match { + case (Some(v1), Some(v2)) => + require(v1.length == v2.length) + val dotProduct = v1.zip(v2).map { case (a, b) => a * b }.sum + dotProduct > threshold + case _ => false + } + } + + def conditionMatch( + action: ci.Action, + candidate: CandidateWithFeatures[TweetCandidate], + topicMap: Map[String, Seq[Double]], + threshold: Double + ): Boolean = { + val now = LocalDate.now() + val currentTime = now.toDate.toInstant.getEpochSecond + val currentDayOfWeek = now.getDayOfWeek + val currentHourOfDay = now.toDateTimeAtCurrentTime.toDateTime.getHourOfDay + + val conditions: Seq[ci.Condition => Boolean] = + Seq( + _.postTopic.forall { topic => + isDotProductClose( + topicMap.get(topic), + candidate.features.getOrElse(TweetTextV8EmbeddingFeature, None), + threshold) + }, + _.postLanguage.forall(candidate.features.getOrElse(TweetLanguageFeature, None).contains), + _.postHasVideo.forall(_ == candidate.features.getOrElse(HasVideoFeature, false)), + _.postHasImage.forall(_ == candidate.features.getOrElse(HasImageFeature, false)), + _.postIsReply.forall( + _ == candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined), + _.postIsRetweet.forall(_ == candidate.features.getOrElse(IsRetweetFeature, false)), + _.postMaximumAge.forall(maxAge => + SnowflakeId.timeFromIdOpt(candidate.candidate.id).exists(_.untilNow.inMinutes <= maxAge)), + _.userFollowsAuthor.forall(_ == candidate.features.get(InNetworkFeature)), + _.postLikesCountGreaterThan.forall(count => + candidate.features + .getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= count))), + _.postLikesCountLessThan.forall(count => + candidate.features + .getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ < count))), + _.postRetweetsCountGreaterThan.forall(count => + candidate.features + .getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ >= count))), + _.postRetweetsCountLessThan.forall(count => + candidate.features + .getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ < count))), + _.postRepliesCountGreaterThan.forall(count => + candidate.features + .getOrElse(EarlybirdFeature, None).exists(_.replyCountV2.exists(_ >= count))), + _.postRepliesCountLessThan.forall(count => + candidate.features + .getOrElse(EarlybirdFeature, None).exists(_.replyCountV2.exists(_ < count))), + _.postQuotesCountGreaterThan.forall(count => + candidate.features + .getOrElse(EarlybirdFeature, None).exists(_.quoteCount.exists(_ >= count))), + _.postQuotesCountLessThan.forall(count => + candidate.features + .getOrElse(EarlybirdFeature, None).exists(_.quoteCount.exists(_ < count))), + _.postLikedByFollowings.forall( + _ == candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty).nonEmpty), + _.authorId.forall(aid => candidate.features.getOrElse(AuthorIdFeature, None).contains(aid)), + _.authorLanguage.forall(lang => + candidate.features.getOrElse(TweetLanguageFeature, None).contains(lang)), + _.authorAccountAgeGreaterThan.forall(count => + candidate.features.getOrElse(AuthorAccountAge, None).exists(_.inDays / 365 >= count)), + _.authorAccountAgeLessThan.forall(count => + candidate.features.getOrElse(AuthorAccountAge, None).exists(_.inDays / 365 < count)), + _.authorFollowerCountGreaterThan.forall(count => + candidate.features.getOrElse(AuthorFollowersFeature, None).exists(_ >= count)), + _.authorFollowerCountLessThan.forall(count => + candidate.features.getOrElse(AuthorFollowersFeature, None).exists(_ < count)), + _.authorFollowedByFollowings.forall( + _ == candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty), + _.queryValidStartTime.forall(currentTime >= _), + _.queryValidEndTime.forall(currentTime < _), + _.queryValidDayOfWeek.forall(_.toInt == currentDayOfWeek), + _.queryValidHourOfDay.forall(_.toInt == currentHourOfDay) + ) + + conditions.forall(_(action.condition)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/BUILD.bazel index c1b5032db..031b6d9d8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/BUILD.bazel @@ -4,22 +4,10 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", - "finagle/finagle-memcached/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", @@ -28,51 +16,25 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/cursor/timelines", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "src/java/com/twitter/search/common/schema/base", "src/java/com/twitter/search/common/schema/earlybird", - "src/java/com/twitter/search/common/util/lang", "src/java/com/twitter/search/queryparser/query:core-query-nodes", "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/tweetypie:service-scala", - "stitch/stitch-gizmoduck", "stitch/stitch-tweetypie", - "stringcenter/client", - "stringcenter/client/src/main/java", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", ], exports = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdCandidatePipelineConfig.scala index c331f955b..ef1b13076 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdCandidatePipelineConfig.scala @@ -1,8 +1,8 @@ package com.twitter.home_mixer.product.subscribed import com.google.inject.Inject -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery +import com.twitter.product_mixer.component_library.candidate_source.earlybird.EarlybirdTweetCandidateSource import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSSubscribedUsersFeature import com.twitter.product_mixer.component_library.filter.TweetVisibilityFilter import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate @@ -21,7 +21,7 @@ import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} import com.twitter.tweetypie.thriftscala.TweetVisibilityPolicy class SubscribedEarlybirdCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, + earlybirdTweetCandidateSource: EarlybirdTweetCandidateSource, tweetyPieStitchClient: TweetypieStitchClient, subscribedEarlybirdQueryTransformer: SubscribedEarlybirdQueryTransformer) extends CandidatePipelineConfig[ @@ -34,7 +34,7 @@ class SubscribedEarlybirdCandidatePipelineConfig @Inject() ( CandidatePipelineIdentifier("SubscribedEarlybird") override val candidateSource: BaseCandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] = - earlybirdCandidateSource + earlybirdTweetCandidateSource override val gates: Seq[Gate[SubscribedQuery]] = Seq( NonEmptySeqFeatureGate(SGSSubscribedUsersFeature) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedMixerPipelineConfig.scala index 5b1f74b37..3aa25117d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedMixerPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedMixerPipelineConfig.scala @@ -1,33 +1,33 @@ package com.twitter.home_mixer.product.subscribed import com.twitter.clientapp.{thriftscala => ca} +import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig -import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.feature_hydrator._ import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration import com.twitter.home_mixer.functional_component.side_effect._ -import com.twitter.home_mixer.model.GapIncludeInstruction import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery +import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.EnablePostContextFeatureHydratorParam import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.ServerMaxResultsParam import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.location.UserLocationQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSSubscribedUsersQueryFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesWithReplaceAndShowAlertInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedCursorIdSelector.TweetIdSelector import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries @@ -35,6 +35,7 @@ import com.twitter.product_mixer.component_library.premarshaller.urt.builder.Rep import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.earlybird.EarlybirdGapIncludeInstruction import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.SelectConditionally @@ -47,13 +48,10 @@ import com.twitter.product_mixer.core.functional_component.marshaller.response.u import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig @@ -71,7 +69,6 @@ class SubscribedMixerPipelineConfig @Inject() ( conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ SubscribedQuery ], - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[SubscribedQuery], dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, @@ -79,12 +76,9 @@ class SubscribedMixerPipelineConfig @Inject() ( requestQueryFeatureHydrator: RequestQueryFeatureHydrator[SubscribedQuery], sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, sgsSubscribedUsersQueryFeatureHydrator: SGSSubscribedUsersQueryFeatureHydrator, - manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ - SubscribedQuery - ], memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, + userLocationQueryFeatureHydrator: UserLocationQueryFeatureHydrator, publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, homeTimelineServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], externalStrings: HomeMixerExternalStrings, @@ -104,7 +98,7 @@ class SubscribedMixerPipelineConfig @Inject() ( sgsSubscribedUsersQueryFeatureHydrator, AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), + AsyncQueryFeatureHydrator(dependentCandidatesStep, userLocationQueryFeatureHydrator), AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) ) @@ -113,8 +107,9 @@ class SubscribedMixerPipelineConfig @Inject() ( private val conversationServiceCandidatePipelineConfig = conversationServiceCandidatePipelineConfigBuilder.build( - Seq(NonEmptyCandidatesGate(earlybirdCandidatePipelineScope)), - HomeConversationServiceCandidateDecorator(homeFeedbackActionInfoBuilder) + earlybirdCandidatePipelineScope, + hmt.ServedType.Subscribed, + EnablePostContextFeatureHydratorParam ) override val candidatePipelines: Seq[CandidatePipelineConfig[SubscribedQuery, _, _, _]] = @@ -178,8 +173,7 @@ class SubscribedMixerPipelineConfig @Inject() ( override val resultSideEffects: Seq[PipelineResultSideEffect[SubscribedQuery, Timeline]] = Seq( homeScribeClientEventSideEffect, homeTimelineServedCandidatesSideEffect, - publishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect + publishClientSentImpressionsEventBusSideEffect ) override val domainMarshaller: DomainMarshaller[SubscribedQuery, Timeline] = { @@ -189,25 +183,14 @@ class SubscribedMixerPipelineConfig @Inject() ( ShowAlertInstructionBuilder(), ) - val idSelector: PartialFunction[UniversalNoun[_], Long] = { - // exclude ads while determining tweet cursor values - case item: TweetItem if item.promotedMetadata.isEmpty => item.id - case module: TimelineModule - if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => - module.items.last.item match { case item: TweetItem => item.id } - } - - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) + val topCursorBuilder = OrderedTopCursorBuilder(TweetIdSelector) val bottomCursorBuilder = - OrderedBottomCursorBuilder(idSelector, GapIncludeInstruction.inverse()) - val gapCursorBuilder = OrderedGapCursorBuilder(idSelector, GapIncludeInstruction) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig(page = Some("subscribed"), section = None, entityToken = None))) - ) + OrderedBottomCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction.inverse()) + val gapCursorBuilder = OrderedGapCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction) + + val scribeConfigBuilder = + StaticTimelineScribeConfigBuilder(TimelineScribeConfig(Some("subscribed"), None, None)) + val metadataBuilder = UrtMetadataBuilder(scribeConfigBuilder = Some(scribeConfigBuilder)) UrtDomainMarshaller( instructionBuilders = instructionBuilders, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedProductPipelineConfig.scala index 0d524391a..49ea2e314 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedProductPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedProductPipelineConfig.scala @@ -1,7 +1,6 @@ package com.twitter.home_mixer.product.subscribed import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.model.request.SubscribedProductContext @@ -10,6 +9,7 @@ import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.ServerMax import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor +import com.twitter.product_mixer.component_library.premarshaller.cursor.timelines.ChronologicalCursorUnmarshaller import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy import com.twitter.product_mixer.core.functional_component.common.alert.Alert diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/BUILD.bazel index b821a8f8d..3081d1cf2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/BUILD.bazel @@ -4,14 +4,8 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stringcenter/client", - "stringcenter/client/src/main/java", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/BUILD.bazel index a56e3a1fd..b4ff2a9b8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/BUILD.bazel @@ -4,11 +4,7 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "util/util-core/src/main/scala/com/twitter/conversions", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParam.scala index 9e4ade43a..a8fb960ef 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParam.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParam.scala @@ -1,6 +1,7 @@ package com.twitter.home_mixer.product.subscribed.param import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam object SubscribedParam { val SupportedClientFSName = "subscribed_supported_client" @@ -12,4 +13,10 @@ object SubscribedParam { min = 1, max = 500 ) + + object EnablePostContextFeatureHydratorParam + extends FSParam[Boolean]( + name = "subscribed_enable_post_context_feature_hydrator", + default = false + ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParamConfig.scala index 58ce7ec35..ebc0b6334 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParamConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParamConfig.scala @@ -15,4 +15,8 @@ class SubscribedParamConfig @Inject() () extends ProductParamConfig { override val boundedIntFSOverrides = Seq( ServerMaxResultsParam ) + + override val booleanFSOverrides = Seq( + EnablePostContextFeatureHydratorParam + ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/BUILD.bazel index c0211ff72..f66d94806 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/BUILD.bazel @@ -4,12 +4,7 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "stitch/stitch-core", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAccessPolicy.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAccessPolicy.scala index 853f4b56a..7f33353fc 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAccessPolicy.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAccessPolicy.scala @@ -10,4 +10,4 @@ object HomeMixerAccessPolicy { * to have a common policy. */ val DefaultHomeMixerAccessPolicy: Set[AccessPolicy] = Set(AllowedLdapGroups(Set.empty[String])) -} +} \ No newline at end of file diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala index 597fd4d36..93c361516 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala @@ -26,8 +26,7 @@ object HomeMixerAlertConfig { object BusinessHours { val DefaultNotificationGroup: NotificationGroup = NotificationGroup( warn = Destination(emails = Seq("")), - critical = Destination(emails = - Seq("")) + critical = Destination(emails = Seq("")) ) def defaultEmptyResponseRateAlert(warnThreshold: Double = 50, criticalThreshold: Double = 80) = diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/ScoredTweetsService.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/ScoredTweetsService.scala index 158e5ee45..eceeea5b2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/ScoredTweetsService.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/ScoredTweetsService.scala @@ -21,4 +21,5 @@ class ScoredTweetsService @Inject() (productPipelineRegistry: ProductPipelineReg ): Stitch[t.ScoredTweetsResponse] = productPipelineRegistry .getProductPipeline[RequestType, t.ScoredTweetsResponse](request.product) .process(ProductPipelineRequest(request, params)) + } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel index c4855d9e7..3ef8b35df 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel @@ -6,8 +6,25 @@ scala_library( dependencies = [ "3rdparty/jvm/com/twitter/bijection:scrooge", "3rdparty/jvm/com/twitter/storehaus:core", + "finagle/finagle-memcached/src/main/scala", + "finatra/inject/inject-core/src/main/scala", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/offheap", + "media-understanding/video-summary/thrift/src/main/thrift:thrift-scala", + "src/java/com/twitter/ml", + "src/scala/com/twitter/ml/api:api-base", + "src/scala/com/twitter/ml/api/util:datarecord", + "src/scala/com/twitter/simclusters_v2/summingbird/stores", + "src/scala/com/twitter/storehaus_internal/manhattan", + "src/scala/com/twitter/storehaus_internal/memcache", + "src/scala/com/twitter/storehaus_internal/memcache/config", + "src/scala/com/twitter/storehaus_internal/offline", + "src/scala/com/twitter/storehaus_internal/util", + "src/thrift/com/twitter/ml/api:data-java", + "src/thrift/com/twitter/timelines/realtime_aggregates:thrift-scala", + "src/thrift/com/twitter/twistly:twistly-scala", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", "stitch/stitch-core", - "storage/clients/manhattan/client/src/main/scala", + "util-internal/util-cache/src/main/java", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterId88Store.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterId88Store.scala new file mode 100644 index 000000000..eda5d9a8f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterId88Store.scala @@ -0,0 +1,19 @@ +package com.twitter.home_mixer.store + +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class MediaClusterId88Store @Inject() ( + val statsReceiver: StatsReceiver, + val serviceIdentifier: ServiceIdentifier) + extends MediaClusterIdStoreTrait { + + override val appId: String = "media_understanding_embeddings_prod" + override val dataset: String = "media_embedding_twitter_clip_v0_cluster_id_88" + override val memcacheDest: String = "/s/cache/clip_cluster_id_88" + override val keyPrefix: String = "" // Empty key prefix as per config + override val storeName: String = "MediaClusterId88Store" +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterId95Store.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterId95Store.scala new file mode 100644 index 000000000..69b9b75ca --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterId95Store.scala @@ -0,0 +1,19 @@ +package com.twitter.home_mixer.store + +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class MediaClusterId95Store @Inject() ( + override val statsReceiver: StatsReceiver, + override val serviceIdentifier: ServiceIdentifier) + extends MediaClusterIdStoreTrait { + + override def appId: String = "media_understanding_embeddings_prod" + override def dataset: String = "media_embedding_twitter_clip_v0_cluster_id_95" + override def memcacheDest: String = "/s/cache/clip_cluster_id_95" + override def keyPrefix: String = "" // Empty key prefix as per config + override def storeName: String = "MediaClusterId95Store" +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterIdStoreTrait.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterIdStoreTrait.scala new file mode 100644 index 000000000..a375b96da --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterIdStoreTrait.scala @@ -0,0 +1,102 @@ +package com.twitter.home_mixer.store + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hermit.store.common.ObservedMemcachedReadableStore +import com.twitter.storage.client.manhattan.bijections.Bijections +import com.twitter.storage.client.manhattan.kv.ManhattanKVClient +import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams +import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder +import com.twitter.storage.client.manhattan.kv.impl.ReadOnlyKeyDescriptor +import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor +import com.twitter.storehaus.ReadableStore +import com.twitter.storehaus_internal.memcache.MemcacheStore +import com.twitter.storehaus_internal.util.ClientName +import com.twitter.storehaus_internal.util.ZkEndPoint +import com.twitter.bijection.Bijection + +trait MediaClusterIdStoreTrait { + val statsReceiver: StatsReceiver + val serviceIdentifier: ServiceIdentifier + + // Abstract methods that subclasses must implement + def appId: String + def dataset: String + def memcacheDest: String + def keyPrefix: String + def storeName: String // For logging + + private val ManhattanDest = "/s/manhattan/nash.native-thrift" + + lazy val clusterIdStore: ReadableStore[Long, Long] = { + val manhattanStore = createManhattanStore() + val cachedStore = createCachedStore(manhattanStore) + cachedStore + } + + private def createCachedStore( + underlyingStore: ReadableStore[Long, Long] + ): ReadableStore[Long, Long] = { + val memcacheStats = statsReceiver.scope(s"memcache_${keyPrefix}") + val underlyingCacheClient = MemcacheStore.memcachedClient( + name = ClientName(serviceIdentifier.name), + dest = ZkEndPoint(memcacheDest), + statsReceiver = memcacheStats, + serviceIdentifier = serviceIdentifier, + timeout = 80.milliseconds + ) + + val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient( + backingStore = underlyingStore, + cacheClient = underlyingCacheClient, + ttl = 3.days, // From config: foundTtl = 3.days + )( + valueInjection = Bijections.long2ByteArray, + statsReceiver = memcacheStats, + keyToString = { key: Long => + val cacheKey = if (keyPrefix.isEmpty) key.toString else s"${keyPrefix}_${key}" + cacheKey + } + ) + + memcacheStore + } + + private def createManhattanStore(): ReadableStore[Long, Long] = { + + // Use proper key and value descriptors with correct encodings + // Sign flip for NativeEncoding (flip high bit to match server-side ordering) + def signFlip(a: Array[Byte]): Array[Byte] = { a(0) = (a(0) ^ 0x80.toByte).toByte; a } + val signBijection = Bijection.build[Array[Byte], Array[Byte]](signFlip _)(signFlip _) + + val keyInjection = + Bijections.long2ByteArray.andThen(signBijection).andThen(Bijections.byteArray2Buf) + val keyDesc = ReadOnlyKeyDescriptor(keyInjection) + val datasetKey = keyDesc.withDataset(dataset) + + val valueInjection = Bijections.long2ByteArray.andThen(Bijections.byteArray2Buf) + val valueDesc = ValueDescriptor(valueInjection) + + val client = ManhattanKVClient( + appId = appId, + dest = ManhattanDest, + mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), + label = storeName + ) + + val endpoint = ManhattanKVEndpointBuilder(client) + .maxRetryCount(3) + .defaultMaxTimeout(120.milliseconds) + .build() + + new ReadableStore[Long, Long] { + override def get(key: Long) = { + import com.twitter.stitch.Stitch + val future = Stitch.run(endpoint.get(datasetKey.withPkey(key), valueDesc)) + future.map(_.map(_.contents)) + } + } + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RTAMHStore.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RTAMHStore.scala new file mode 100644 index 000000000..448aa4c8d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RTAMHStore.scala @@ -0,0 +1,39 @@ +package com.twitter.home_mixer.store + +import com.twitter.bijection.Injection +import com.twitter.bijection.scrooge.BinaryScalaCodec +import com.twitter.home_mixer.store.RTAManhattanRealGraphKVDescriptor._ +import com.twitter.io.Buf +import com.twitter.ml.api.DataRecord +import com.twitter.stitch.Stitch +import com.twitter.storage.client.manhattan.bijections.Bijections +import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint +import com.twitter.storage.client.manhattan.kv.impl.ReadOnlyKeyDescriptor +import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future +import com.twitter.ml.api.{thriftscala => mlThrift} +import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} +import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions._ + +object RTAManhattanRealGraphKVDescriptor { + val datasetName = "timelines_real_time_aggregates_0" + + val keyInjection: Injection[thrift.AggregationKey, Buf] = + BinaryScalaCodec(thrift.AggregationKey).andThen(Bijections.byteArray2Buf) + val keyDesc = ReadOnlyKeyDescriptor(keyInjection) + val datasetKey = keyDesc.withDataset(datasetName) + val valueInjection = BinaryScalaCodec(mlThrift.DataRecord).andThen(Bijections.byteArray2Buf) + val valueDesc = ValueDescriptor(valueInjection) +} + +/** + * + */ +class RTAMHStore(manhattanKVEndpoint: ManhattanKVEndpoint) + extends ReadableStore[thrift.AggregationKey, DataRecord] { + + override def get(key: thrift.AggregationKey): Future[Option[DataRecord]] = Stitch + .run(manhattanKVEndpoint.get(datasetKey.withPkey(key), valueDesc)) + .map(_.map(mhResponse => mhResponse.contents).map(scalaDataRecord2JavaDataRecord)) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RealGraphInNetworkScoresStore.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RealGraphInNetworkScoresStore.scala index ce0b182be..bd5f6b897 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RealGraphInNetworkScoresStore.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RealGraphInNetworkScoresStore.scala @@ -15,7 +15,7 @@ import com.twitter.wtf.candidate.{thriftscala => wtf} object ManhattanRealGraphKVDescriptor { implicit val byteArray2Buf = Bijections.BytesBijection - val realGraphDatasetName = "real_graph_scores_in" + val realGraphDatasetName = "real_graph_scores_in_v1" val keyInjection = Injection.connect[Long, Array[Byte]].andThen(Bijections.BytesInjection) val keyDesc = ReadOnlyKeyDescriptor(keyInjection) val valueDesc = ValueDescriptor(BinaryScalaInjection(wtf.CandidateSeq)) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/TweetWatchTimeMetadataStore.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/TweetWatchTimeMetadataStore.scala new file mode 100644 index 000000000..1b91f5416 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/TweetWatchTimeMetadataStore.scala @@ -0,0 +1,122 @@ +package com.twitter.home_mixer.store + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hermit.store.common.ObservedMemcachedReadableStore +import com.twitter.storage.client.manhattan.bijections.Bijections +import com.twitter.storage.client.manhattan.kv.ManhattanKVClient +import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams +import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder +import com.twitter.storage.client.manhattan.kv.impl.Component +import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor +import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor +import com.twitter.storehaus.ReadableStore +import com.twitter.storehaus_internal.memcache.MemcacheStore +import com.twitter.storehaus_internal.util.ClientName +import com.twitter.storehaus_internal.util.ZkEndPoint +import com.twitter.twistly.thriftscala.VideoViewEngagementType +import com.twitter.twistly.thriftscala.WatchTimeMetadata +import com.twitter.bijection.Injection +import com.twitter.bijection.scrooge.BinaryScalaCodec +import com.twitter.io.Buf +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Try + +@Singleton +class TweetWatchTimeMetadataStore @Inject() ( + statsReceiver: StatsReceiver, + serviceIdentifier: ServiceIdentifier) { + + private val ManhattanDest = "/s/manhattan/nash.native-thrift" + private val AppId = "uss_prod" + private val Dataset = "video_watch_time_metadata" + private val MemcacheDest = "/s/cache/tweet_aggregated_watch_time" + private val KeyPrefix = "" // Empty key prefix as per config + + lazy val tweetWatchTimeMetadataStore: ReadableStore[ + (Long, VideoViewEngagementType), + WatchTimeMetadata + ] = { + val manhattanStore = createManhattanStore() + val cachedStore = createCachedStore(manhattanStore) + cachedStore + } + + private def createCachedStore( + underlyingStore: ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata] + ): ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata] = { + val memcacheStats = statsReceiver.scope(s"memcache_${KeyPrefix}") + val underlyingCacheClient = MemcacheStore.memcachedClient( + name = ClientName(serviceIdentifier.name), + dest = ZkEndPoint(MemcacheDest), + statsReceiver = memcacheStats, + serviceIdentifier = serviceIdentifier, + timeout = 80.milliseconds + ) + + val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient( + backingStore = underlyingStore, + cacheClient = underlyingCacheClient, + ttl = 10.minutes, + )( + valueInjection = BinaryScalaCodec(WatchTimeMetadata), + statsReceiver = memcacheStats, + keyToString = { key: (Long, VideoViewEngagementType) => + val cacheKey = + if (KeyPrefix.isEmpty) s"${key._1}_${key._2}" else s"${KeyPrefix}_${key._1}_${key._2}" + cacheKey + } + ) + + memcacheStore + } + + private def createManhattanStore( + ): ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata] = { + + val pkeyInjection: Injection[Long, Buf] = + Bijections.long2ByteArray.andThen(Bijections.BytesBijection) + + val lkeyInjection: Injection[VideoViewEngagementType, Buf] = + Injection + .build[VideoViewEngagementType, Int](_.value)(i => Try(VideoViewEngagementType(i))) + .andThen(Bijections.int2ByteArray) + .andThen(Bijections.BytesBijection) + + val keyDesc = + KeyDescriptor(Component(pkeyInjection), Component(lkeyInjection)).withDataset(Dataset) + + val valueInjection: Injection[WatchTimeMetadata, Buf] = + Bijections.BinaryScalaInjection(WatchTimeMetadata) + + val valueDesc = ValueDescriptor(valueInjection) + + val client = ManhattanKVClient( + appId = AppId, + dest = ManhattanDest, + mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), + label = "TweetWatchTimeMetadata" + ) + + val endpoint = ManhattanKVEndpointBuilder(client) + .maxRetryCount(3) + .defaultMaxTimeout(120.milliseconds) + .build() + + new ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata] { + override def get(key: (Long, VideoViewEngagementType)) = { + val (tweetId, engType) = key + val mhKey = keyDesc.withPkey(tweetId).withLkey(engType) + Stitch.run(endpoint.get(mhKey, valueDesc)).map { opt => + opt.map { mv => + mv.contents + } + } + } + } + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/TwhinEmbeddingsStore.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/TwhinEmbeddingsStore.scala new file mode 100644 index 000000000..91bd68813 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/TwhinEmbeddingsStore.scala @@ -0,0 +1,195 @@ +package com.twitter.home_mixer.store + +import com.twitter.bijection.scrooge.BinaryScalaCodec +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hermit.store.common.ObservedMemcachedReadableStore +import com.twitter.hermit.store.offheap.Codecs +import com.twitter.hermit.store.offheap.OffheapCachedReadableStore +import com.twitter.scrooge.ThriftStruct +import com.twitter.simclusters_v2.summingbird.stores.ManhattanFromStratoStore +import com.twitter.simclusters_v2.thriftscala.PersistentTwhinTweetEmbedding +import com.twitter.simclusters_v2.thriftscala.PersistentTwhinUserEmbedding +import com.twitter.simclusters_v2.{thriftscala => sim} +import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams +import com.twitter.storehaus.ReadableStore +import com.twitter.storehaus_internal.memcache.MemcacheStore +import com.twitter.storehaus_internal.util._ +import javax.inject.Inject +import javax.inject.Singleton +import com.twitter.offheap.KeyHashFunction +import com.twitter.offheap.MemoryWriter +import com.twitter.offheap.ObjectCodec +import com.twitter.offheap.OffheapReader +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding + +@Singleton() +class TwhinEmbeddingsStore @Inject() ( + statsReceiver: StatsReceiver, + serviceIdentifier: ServiceIdentifier) { + + val ManhattanNashDest = "/s/manhattan/nash.native-thrift" + val TwhinEmbeddingsProdAppId = "twhin_embeddings_prod" + val UserPositiveDataset = "twhin_user_positive_embeddings" + val RebuildUserPositiveDataset = "twhin_rebuild_user_rt_pos_emb" + val UserNegativeDataset = "twhin_user_negative_embeddings" + val TweetDataset = "twhin_tweet_embeddings" + val TweetRebuildDataset = "twhin_rebuild_tweet_rt_emb" + val VideoDataset = "twhin_video_embeddings" + + val MemcacheTweetDest = "/s/cache/twhin_embeddings" + val MemcacheVideoDest = "/s/cache/twhin_video_embeddings" + val KeyPrefixTweet = "twhin_tweets" + val KeyPrefixTweetRebuild = "twhin_tweets_rebuild" + val KeyPrefixVideo = "twhin_videos" + val InMemoryCachePrefix = "in_memory_cache" + + val MinEngagementCount = 16 + val IsProdEnv = serviceIdentifier.environment == "prod" + + /** + * We do not generate the tweet or video embedding if the number of recent engagements + * is < `MinEngagementCount`. This is based on prior Simcluster embedding aggregation + * experience and in order to be consistent with the Strato column + * strato/config/columns/recommendations/twhin/CachedTwhinTweetEmbeddings.Tweet.strato + */ + private def normalizeByCount( + persistentEmbedding: sim.PersistentTwhinTweetEmbedding + ): sim.TwhinTweetEmbedding = { + val embedding = persistentEmbedding.embedding.embedding + val updatedEmbedding = + if (persistentEmbedding.updatedCount < MinEngagementCount) embedding.map(_ => 0.0) + else embedding.map(_ / persistentEmbedding.updatedCount) + sim.TwhinTweetEmbedding(updatedEmbedding) + } + + private def createManhattanStore[T <: ThriftStruct: Manifest]( + dataset: String + ): ReadableStore[Long, T] = { + ManhattanFromStratoStore + .createPersistentTwhinStore[T]( + dataset = dataset, + mhMtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), + statsReceiver = statsReceiver, + appId = TwhinEmbeddingsProdAppId, + dest = ManhattanNashDest + ).composeKeyMapping((_, 0L)) + } + + private def createManhattanVersionedStore[T <: ThriftStruct: Manifest]( + dataset: String + ): ReadableStore[(Long, Long), T] = { + ManhattanFromStratoStore + .createPersistentTwhinStore[T]( + dataset = dataset, + mhMtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), + statsReceiver = statsReceiver, + appId = TwhinEmbeddingsProdAppId, + dest = ManhattanNashDest + ).composeKeyMapping[(Long, Long)](key => key) + } + + private def createCachedStore[K]( + underlyingStore: ReadableStore[K, sim.TwhinTweetEmbedding], + cacheDest: String, + keyPrefix: String, + keyCodec: ObjectCodec[K], + keyHashFunction: KeyHashFunction[K] + ): ReadableStore[K, sim.TwhinTweetEmbedding] = { + val scopedStatsReceiver = statsReceiver.scope(keyPrefix) + val underlyingCacheClient = MemcacheStore.memcachedClient( + name = ClientName(keyPrefix), + dest = ZkEndPoint(cacheDest), + statsReceiver = scopedStatsReceiver, + serviceIdentifier = serviceIdentifier, + timeout = 80.milliseconds + ) + + val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient( + backingStore = underlyingStore, + cacheClient = underlyingCacheClient, + ttl = 15.minutes, + asyncUpdate = IsProdEnv + )( + valueInjection = BinaryScalaCodec(sim.TwhinTweetEmbedding), + statsReceiver = scopedStatsReceiver, + keyToString = { key: K => s"${keyPrefix}_${key}" } + ) + + OffheapCachedReadableStore.fromCache( + memcacheStore, + tableSize = 256 * 1024, + capacity = 256 * 1024 * 2048, + keyHashFunction = keyHashFunction, + keyCodec = keyCodec, + valueCodec = TwhinEmbeddingsStore.TwhinTweetEmbeddingCodec, + ttl = 1.minutes, + statsReceiver = scopedStatsReceiver + ) + } + + val mhUserPositiveStore: ReadableStore[Long, sim.TwhinTweetEmbedding] = + createManhattanStore[PersistentTwhinUserEmbedding](UserPositiveDataset).mapValues(_.embedding) + + val mhRebuildUserPositiveStore: ReadableStore[(Long, Long), sim.TwhinTweetEmbedding] = + createManhattanVersionedStore[PersistentTwhinUserEmbedding](RebuildUserPositiveDataset) + .mapValues(_.embedding) + + val mhUserNegativeStore: ReadableStore[Long, sim.TwhinTweetEmbedding] = + createManhattanStore[PersistentTwhinUserEmbedding](UserNegativeDataset).mapValues(_.embedding) + + val mhTweetStore: ReadableStore[Long, sim.TwhinTweetEmbedding] = + createManhattanStore[PersistentTwhinTweetEmbedding](TweetDataset).mapValues(normalizeByCount) + + val mhVideoStore: ReadableStore[Long, sim.TwhinTweetEmbedding] = + createManhattanStore[PersistentTwhinTweetEmbedding](VideoDataset).mapValues(normalizeByCount) + + val cachedTweetStore: ReadableStore[Long, TwhinTweetEmbedding] = createCachedStore( + mhTweetStore, + MemcacheTweetDest, + KeyPrefixTweet, + Codecs.LongKeyCodec, + key => java.lang.Long.hashCode(key) + ) + + val mhTweetRebuildStore: ReadableStore[(Long, Long), sim.TwhinTweetEmbedding] = + createManhattanVersionedStore[PersistentTwhinTweetEmbedding](TweetRebuildDataset) + .mapValues(normalizeByCount) + + val cachedTweetRebuildStore: ReadableStore[(Long, Long), TwhinTweetEmbedding] = + createCachedStore[(Long, Long)]( + mhTweetRebuildStore, + MemcacheTweetDest, + KeyPrefixTweetRebuild, + Codecs.LongTupleKeyCodec, + key => key.hashCode() + ) + + val cachedVideoStore: ReadableStore[Long, TwhinTweetEmbedding] = createCachedStore( + mhVideoStore, + MemcacheVideoDest, + KeyPrefixVideo, + Codecs.LongKeyCodec, + key => java.lang.Long.hashCode(key) + ) +} + +object TwhinEmbeddingsStore { + object TwhinTweetEmbeddingCodec extends ObjectCodec[sim.TwhinTweetEmbedding] { + override def size(decoded: sim.TwhinTweetEmbedding): Int = { + Codecs.SeqDoubleCodec.size(decoded.embedding) + } + + override def encode(decoded: sim.TwhinTweetEmbedding, writer: MemoryWriter): Unit = { + Codecs.SeqDoubleCodec.encode(decoded.embedding, writer) + } + + override def decode(reader: OffheapReader): sim.TwhinTweetEmbedding = { + + sim.TwhinTweetEmbedding( + embedding = Codecs.SeqDoubleCodec.decode(reader) + ) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/VideoEmbeddingMHStore.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/VideoEmbeddingMHStore.scala new file mode 100644 index 000000000..ceeb997ea --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/VideoEmbeddingMHStore.scala @@ -0,0 +1,106 @@ +package com.twitter.home_mixer.store + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hermit.store.common.ObservedMemcachedReadableStore +import com.twitter.storage.client.manhattan.bijections.Bijections +import com.twitter.storage.client.manhattan.kv.ManhattanKVClient +import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams +import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder +import com.twitter.storage.client.manhattan.kv.impl.ReadOnlyKeyDescriptor +import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor +import com.twitter.storehaus.ReadableStore +import com.twitter.storehaus_internal.memcache.MemcacheStore +import com.twitter.storehaus_internal.util.ClientName +import com.twitter.storehaus_internal.util.ZkEndPoint +import com.twitter.media_understanding.video_summary.thriftscala.VideoEmbedding +import com.twitter.bijection.Injection +import com.twitter.bijection.scrooge.BinaryScalaCodec +import com.twitter.io.Buf +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class VideoEmbeddingMHStore @Inject() ( + statsReceiver: StatsReceiver, + serviceIdentifier: ServiceIdentifier) { + + private val ManhattanDest = "/s/manhattan/nash.native-thrift" + private val AppId = "media_understanding_video_summary" + private val Dataset = "video_summary_embedding" + private val MemcacheDest = "/s/cache/video_summary_embedding" + private val KeyPrefix = "summary_embedding" + + lazy val videoEmbeddingMHStore: ReadableStore[Long, VideoEmbedding] = { + val manhattanStore = createManhattanStore() + val cachedStore = createCachedStore(manhattanStore) + cachedStore + } + + private def createCachedStore( + underlyingStore: ReadableStore[Long, VideoEmbedding] + ): ReadableStore[Long, VideoEmbedding] = { + val memcacheStats = statsReceiver.scope(s"memcache_${KeyPrefix}") + val underlyingCacheClient = MemcacheStore.memcachedClient( + name = ClientName(serviceIdentifier.name), + dest = ZkEndPoint(MemcacheDest), + statsReceiver = memcacheStats, + serviceIdentifier = serviceIdentifier, + timeout = 80.milliseconds + ) + + val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient( + backingStore = underlyingStore, + cacheClient = underlyingCacheClient, + ttl = 7.days, // foundTtl from config + )( + valueInjection = BinaryScalaCodec(VideoEmbedding), + statsReceiver = memcacheStats, + keyToString = { key: Long => + s"${KeyPrefix}_${key}" + } + ) + + memcacheStore + } + + private def createManhattanStore(): ReadableStore[Long, VideoEmbedding] = { + + val keyInjection: Injection[Long, Buf] = + Bijections.long2ByteArray.andThen(Bijections.BytesBijection) + + val keyDesc = ReadOnlyKeyDescriptor(keyInjection) + val datasetKey = keyDesc.withDataset(Dataset) + + val valueInjection: Injection[VideoEmbedding, Buf] = + Bijections.BinaryScalaInjection(VideoEmbedding) + + val valueDesc = ValueDescriptor(valueInjection) + + val client = ManhattanKVClient( + appId = AppId, + dest = ManhattanDest, + mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), + label = "VideoEmbeddingMH" + ) + + val endpoint = ManhattanKVEndpointBuilder(client) + .maxRetryCount(3) + .defaultMaxTimeout(120.milliseconds) + .build() + + new ReadableStore[Long, VideoEmbedding] { + override def get(key: Long) = { + val mhKey = datasetKey.withPkey(key) + Stitch.run(endpoint.get(mhKey, valueDesc)).map { opt => + opt.map { mv => + mv.contents + } + } + } + } + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel index 8e04fa11f..439ba0c68 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel @@ -4,22 +4,25 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ + "3rdparty/jvm/io/grpc:grpc-protobuf", + "3rdparty/jvm/io/grpc:grpc-stub", + "finagle-internal/finagle-grpc/src/main/scala", + "finatra/inject/inject-utils/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "servo/repo/src/main/scala", "snowflake/src/main/scala/com/twitter/snowflake/id", "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/api/util", "src/java/com/twitter/search/common/util/lang", "src/scala/com/twitter/ml/api/util", - "src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/search/common:constants-scala", - "src/thrift/com/twitter/service/metastore/gen:thrift-java", "src/thrift/com/twitter/service/metastore/gen:thrift-scala", "storage/clients/manhattan/client/src/main/scala", + "strato/config/src/thrift/com/twitter/strato/columns/content_understanding:content_understanding-scala", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + "user_history_transformer/service/src/main/java/com/x/user_action_sequence", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala index e0fbdd76f..c09e22c85 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala @@ -2,6 +2,7 @@ package com.twitter.home_mixer.util import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature import com.twitter.home_mixer.{thriftscala => hmt} +import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweets import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.snowflake.id.SnowflakeId @@ -13,7 +14,7 @@ object CachedScoredTweetsHelper { features: FeatureMap, candidatePipelineIdentifier: CandidatePipelineIdentifier ): Seq[Long] = { - val tweetImpressions = TweetImpressionsHelper.tweetImpressions(features) + val tweetImpressions = features.getOrElse(ImpressedTweets, Seq.empty).toSet val cachedScoredTweets = features .getOrElse(CachedScoredTweetsFeature, Seq.empty) .filter { tweet => @@ -41,7 +42,7 @@ object CachedScoredTweetsHelper { def unseenCachedScoredTweets( features: FeatureMap ): Seq[hmt.ScoredTweet] = { - val seenTweetIds = TweetImpressionsHelper.tweetImpressions(features) + val seenTweetIds = features.getOrElse(ImpressedTweets, Seq.empty).toSet features .getOrElse(CachedScoredTweetsFeature, Seq.empty) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala index 06fe5646c..16ec9841e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala @@ -1,7 +1,9 @@ package com.twitter.home_mixer.util +import com.twitter.escherbird.common.thriftscala.QualifiedId import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.GrokVideoMetadataFeature import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature @@ -11,6 +13,7 @@ import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.VideoAspectRatioFeature import com.twitter.product_mixer.component_library.model.candidate.CursorCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -25,6 +28,7 @@ import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandid import scala.reflect.ClassTag object CandidatesUtil { + def getItemCandidates(candidates: Seq[CandidateWithDetails]): Seq[ItemCandidateWithDetails] = { candidates.collect { case item: ItemCandidateWithDetails if !item.isCandidateType[CursorCandidate] => Seq(item) @@ -55,8 +59,28 @@ object CandidatesUtil { def getOriginalTweetId(candidate: CandidateWithFeatures[TweetCandidate]): Long = { if (candidate.features.getOrElse(IsRetweetFeature, false)) candidate.features.getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidate.id) - else - candidate.candidate.id + else candidate.candidate.id + } + + def getOriginalTweetId(candidate: TweetCandidate, features: FeatureMap): Long = { + if (features.getOrElse(IsRetweetFeature, false)) + features.getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.id) + else candidate.id + } + + def getOriginalTweetId(candidate: CandidateWithDetails): Long = { + candidate match { + case ItemCandidateWithDetails(candidate: TweetCandidate, _, feautres) => + getOriginalTweetId(candidate, feautres) + case _ => + throw PipelineFailure(UnexpectedCandidateResult, "Invalid candidate type") + } + } + + def getOriginalTweetId(candidateId: Long, features: FeatureMap): Long = { + if (features.getOrElse(IsRetweetFeature, false)) + features.getOrElse(SourceTweetIdFeature, None).getOrElse(candidateId) + else candidateId } def getOriginalAuthorId(candidateFeatures: FeatureMap): Option[Long] = @@ -68,6 +92,10 @@ object CandidatesUtil { !candidate.features.getOrElse(IsRetweetFeature, false) && candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty + def isOriginalTweet(candidate: CandidateWithDetails): Boolean = + !candidate.features.getOrElse(IsRetweetFeature, false) && + candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty + def getEngagerUserIds( candidateFeatures: FeatureMap ): Seq[Long] = { @@ -92,6 +120,19 @@ object CandidatesUtil { (candidateFeatures.getOrElse(IsRetweetFeature, false) && candidateFeatures.getOrElse(SourceUserIdFeature, None).contains(query.getRequiredUserId)) + def getCandidateTopicAndAspectRatio( + candidate: CandidateWithDetails + ): (Option[QualifiedId], Boolean) = { + val video = candidate.features.getOrElse(GrokVideoMetadataFeature, None) + val optionalQualifiedId = video.flatMap { + _.entities.map { entities => + entities.maxBy(entity => entity.score.getOrElse(0.0))._1 + } + } + val aspectRatio = candidate.features.getOrElse(VideoAspectRatioFeature, None).exists(_ > 1.0) + (optionalQualifiedId, aspectRatio) + } + val reverseChronTweetsOrdering: Ordering[CandidateWithDetails] = Ordering.by[CandidateWithDetails, Long] { case ItemCandidateWithDetails(candidate: TweetCandidate, _, _) => -candidate.id diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageCode.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageCode.scala new file mode 100644 index 000000000..b2a87dbcd --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageCode.scala @@ -0,0 +1,92 @@ +package com.twitter.home_mixer.util + +object LanguageCode { + final val Japanese = "jp" + final val English = "en" + final val Unknown = "zxx" + + val AllowedLanguageCodes = Set( + "art", // Emojis + "qam", // Mentions + "qct", // Cashtag + "qht", // Hashtag + "qme", // Multiple entities + "qst", // Short text + "und", // Undefined + "zxx" // Links + ) + + val languageToISO: Map[String, String] = Map( + "arabic" -> "ar", + "danish" -> "da", + "german" -> "de", + "greek" -> "el", + "english" -> "en", + "esperanto" -> "eo", + "spanish" -> "es", + "persian" -> "fa", + "finnish" -> "fi", + "french" -> "fr", + "hebrew" -> "he", + "hungarian" -> "hu", + "indonesian" -> "id", + "icelandic" -> "is", + "italian" -> "it", + "japanese" -> "ja", + "korean" -> "ko", + "lithuanian" -> "lt", + "dutch" -> "nl", + "norwegian" -> "no", + "polish" -> "pl", + "portuguese" -> "pt", + "russian" -> "ru", + "swedish" -> "sv", + "thai" -> "th", + "urdu" -> "ur", + "chinese" -> "zh", + "turkish" -> "tr", + "tagalog" -> "tl", + "hindi" -> "hi", + "malay" -> "ms", + "amharic" -> "am", + "bengali" -> "bn", + "tibetan" -> "bo", + "dhivehi" -> "dv", + "gujarati" -> "gu", + "armenian" -> "hy", + "inuktitut" -> "iu", + "georgian" -> "ka", + "khmer" -> "km", + "kannada" -> "kn", + "lao" -> "lo", + "malayalam" -> "ml", + "myanmar" -> "my", + "oriya" -> "or", + "panjabi" -> "pa", + "sinhala" -> "si", + "tamil" -> "ta", + "telugu" -> "te", + "vietnamese" -> "vi", + "bulgarian" -> "bg", + "nepali" -> "ne", + "estonian" -> "et", + "haitian" -> "ht", + "latvian" -> "lv", + "slovak" -> "sk", + "slovenian" -> "sl", + "ukrainian" -> "uk", + "basque" -> "eu", + "bosnian" -> "bs", + "catalan" -> "ca", + "croatian" -> "hr", + "czech" -> "cs", + "hindi latin" -> "hi", + "marathi" -> "mr", + "pashto" -> "ps", + "romanian" -> "ro", + "serbian" -> "sr", + "sindhi" -> "sd", + "welsh" -> "cy", + "uyghur" -> "ug" + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/NaviScorerStatsHandler.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/NaviScorerStatsHandler.scala new file mode 100644 index 000000000..b56a143f8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/NaviScorerStatsHandler.scala @@ -0,0 +1,107 @@ +package com.twitter.home_mixer.util + +import com.twitter.finagle.stats.BroadcastStatsReceiver +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.NullStatsReceiver +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.NaviClientConfigFeature +import com.twitter.home_mixer.model.PredictedScoreFeature +import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatures +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelIdParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.util.Memoize + +case class ModelStats(broadcastStatsReceiver: StatsReceiver) { + private val StatsReadabilityMultiplier = 1000 + private val PredictedScoreStatName = f"predictedScore${StatsReadabilityMultiplier}x" + private val MissingScoreStatName = "missingScore" + private val ValidScoreStatName = "validScore" + private val NullStat = NullStatsReceiver.stat("NullStat") + private val NullCounter = NullStatsReceiver.counter("NullCounter") + + val failuresStat: Stat = broadcastStatsReceiver.stat("failures") + val responsesStat: Stat = broadcastStatsReceiver.stat("responses") + val invalidResponsesCounter: Counter = broadcastStatsReceiver.counter("invalidResponses") + val scoreStat: Stat = broadcastStatsReceiver.stat(f"score${StatsReadabilityMultiplier}x") + val negativeFilterCounter: Counter = broadcastStatsReceiver.counter("negativeFiltered") + + private val predictedScoreStats: Map[PredictedScoreFeature, Stat] = PredictedScoreFeatures.map { + scoreFeature => + (scoreFeature, broadcastStatsReceiver.stat(scoreFeature.statName, PredictedScoreStatName)) + }.toMap + + private val validScoreCounters: Map[PredictedScoreFeature, Counter] = PredictedScoreFeatures.map { + scoreFeature => + (scoreFeature, broadcastStatsReceiver.counter(scoreFeature.statName, ValidScoreStatName)) + }.toMap + + private val missingScoreCounters: Map[PredictedScoreFeature, Counter] = + PredictedScoreFeatures.map { scoreFeature => + (scoreFeature, broadcastStatsReceiver.counter(scoreFeature.statName, MissingScoreStatName)) + }.toMap + + def getPredictedScoreStat(scoreFeature: PredictedScoreFeature): Stat = + predictedScoreStats.getOrElse(scoreFeature, NullStat) + + def getValidScoreCounter(scoreFeature: PredictedScoreFeature): Counter = + validScoreCounters.getOrElse(scoreFeature, NullCounter) + + def getMissingScoreCounter(scoreFeature: PredictedScoreFeature): Counter = + missingScoreCounters.getOrElse(scoreFeature, NullCounter) + + def trackPredictedScoreStats( + predictedScoreFeature: PredictedScoreFeature, + predictedScoreOpt: Option[Double] + ): Unit = { + predictedScoreOpt match { + case Some(predictedScore) => + getPredictedScoreStat(predictedScoreFeature) + .add((predictedScore * StatsReadabilityMultiplier).toFloat) + getValidScoreCounter(predictedScoreFeature).incr() + case None => + getMissingScoreCounter(predictedScoreFeature).incr() + } + } +} + +class NaviScorerStatsHandler(statsReceiver: StatsReceiver, scope: String) { + private val scopedStatsReceiver = statsReceiver.scope(scope) + + // Memoize stats object so they are not created per request + private val statsPerModel = Memoize[(String, String, String, String), ModelStats] { + case (product, modelId, modelName, clientId) => + // Collect stats overall, per product, per model, and per client + val broadcastStatsReceiver: StatsReceiver = BroadcastStatsReceiver( + Seq( + scopedStatsReceiver, + scopedStatsReceiver.scope(product), + scopedStatsReceiver.scope(modelId).scope(product), + scopedStatsReceiver.scope(modelName).scope(product), + scopedStatsReceiver.scope(clientId).scope(product) + ) + ) + ModelStats(broadcastStatsReceiver) + } + + /** Retrieve ModelStats for the given query */ + def getModelStats( + query: PipelineQuery + ): ModelStats = { + val modelId = query.params(ModelIdParam) + val modelName = query.params(ModelNameParam) + val modelNameStr = if (modelName.nonEmpty) modelName else "EMPTY_MODEL_NAME" + + val naviClientConfig = + query.features.map(_.get(NaviClientConfigFeature)).get // Should always be present + + val clientId = query.clientContext.appId.getOrElse(0L).toString + statsPerModel( + ( + query.product.identifier.toString, + modelId + naviClientConfig.clusterStr, + modelNameStr + naviClientConfig.clusterStr, + clientId)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ObservedKeyValueResultHandler.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ObservedKeyValueResultHandler.scala index 0cd0cd60b..d3a176e90 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ObservedKeyValueResultHandler.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ObservedKeyValueResultHandler.scala @@ -13,7 +13,7 @@ trait ObservedKeyValueResultHandler { private lazy val scopedStatsReceiver = statsReceiver.scope(statScope) private lazy val keyTotalCounter = scopedStatsReceiver.counter("key/total") private lazy val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private lazy val keyLossCounter = scopedStatsReceiver.counter("key/loss") + private lazy val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private lazy val keyFailureCounter = scopedStatsReceiver.counter("key/failure") def observedGet[K, V]( @@ -27,7 +27,7 @@ trait ObservedKeyValueResultHandler { keyFoundCounter.incr() Return(Some(value)) case Return(None) => - keyLossCounter.incr() + keyNotFoundCounter.incr() Return(None) case Throw(exception) => keyFailureCounter.incr() diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/PhoenixScorerStatsHandler.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/PhoenixScorerStatsHandler.scala new file mode 100644 index 000000000..3595bf064 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/PhoenixScorerStatsHandler.scala @@ -0,0 +1,88 @@ +package com.twitter.home_mixer.util + +import com.twitter.finagle.stats.BroadcastStatsReceiver +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.NullStatsReceiver +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature +import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature.PhoenixPredictedScoreFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.util.Memoize + +case class PhoenixModelStats(broadcastStatsReceiver: StatsReceiver) { + private val StatsReadabilityMultiplier = 1000 + private val PredictedScoreStatName = f"predictedScore${StatsReadabilityMultiplier}x" + private val MissingScoreStatName = "missingScore" + private val ValidScoreStatName = "validScore" + private val NullStat = NullStatsReceiver.stat("NullStat") + private val NullCounter = NullStatsReceiver.counter("NullCounter") + + val failuresStat: Stat = broadcastStatsReceiver.stat("failures") + val responsesStat: Stat = broadcastStatsReceiver.stat("responses") + val invalidResponsesCounter: Counter = broadcastStatsReceiver.counter("invalidResponses") + val scoreStat: Stat = broadcastStatsReceiver.stat(f"score${StatsReadabilityMultiplier}x") + val negativeFilterCounter: Counter = broadcastStatsReceiver.counter("negativeFiltered") + + private val predictedScoreStats: Map[PhoenixPredictedScoreFeature, Stat] = + PhoenixPredictedScoreFeatures.map { scoreFeature => + (scoreFeature, broadcastStatsReceiver.stat(scoreFeature.featureName, PredictedScoreStatName)) + }.toMap + + private val validScoreCounters: Map[PhoenixPredictedScoreFeature, Counter] = + PhoenixPredictedScoreFeatures.map { scoreFeature => + (scoreFeature, broadcastStatsReceiver.counter(scoreFeature.featureName, ValidScoreStatName)) + }.toMap + + private val missingScoreCounters: Map[PhoenixPredictedScoreFeature, Counter] = + PhoenixPredictedScoreFeatures.map { scoreFeature => + (scoreFeature, broadcastStatsReceiver.counter(scoreFeature.featureName, MissingScoreStatName)) + }.toMap + + def getPredictedScoreStat(scoreFeature: PhoenixPredictedScoreFeature): Stat = + predictedScoreStats.getOrElse(scoreFeature, NullStat) + + def getValidScoreCounter(scoreFeature: PhoenixPredictedScoreFeature): Counter = + validScoreCounters.getOrElse(scoreFeature, NullCounter) + + def getMissingScoreCounter(scoreFeature: PhoenixPredictedScoreFeature): Counter = + missingScoreCounters.getOrElse(scoreFeature, NullCounter) + + def trackPredictedScoreStats( + predictedScoreFeature: PhoenixPredictedScoreFeature, + predictedScoreOpt: Option[Double] + ): Unit = { + predictedScoreOpt match { + case Some(predictedScore) => + getPredictedScoreStat(predictedScoreFeature) + .add((predictedScore * StatsReadabilityMultiplier).toFloat) + getValidScoreCounter(predictedScoreFeature).incr() + case None => + getMissingScoreCounter(predictedScoreFeature).incr() + } + } +} + +class PhoenixScorerStatsHandler(statsReceiver: StatsReceiver, scope: String) { + private val scopedStatsReceiver = statsReceiver.scope(scope) + + // Memoize stats object so they are not created per request + private val statsPerModel = Memoize[(String, String, String), PhoenixModelStats] { + case (product, clientId, cluster) => + // Collect stats overall, per product, and per client + val broadcastStatsReceiver: StatsReceiver = BroadcastStatsReceiver( + Seq( + scopedStatsReceiver, + scopedStatsReceiver.scope(product), + scopedStatsReceiver.scope(product).scope(cluster), + scopedStatsReceiver.scope(clientId).scope(product) + ) + ) + PhoenixModelStats(broadcastStatsReceiver) + } + + def getModelStats(query: PipelineQuery, cluster: String): PhoenixModelStats = { + val clientId = query.clientContext.appId.getOrElse(0L).toString + statsPerModel((query.product.identifier.toString, clientId, cluster)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/PhoenixUtils.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/PhoenixUtils.scala new file mode 100644 index 000000000..83fefd14a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/PhoenixUtils.scala @@ -0,0 +1,161 @@ +package com.twitter.home_mixer.util + +import com.twitter.finagle.grpc.FutureConverters +import com.twitter.finagle.service.RetryPolicy +import com.twitter.finagle.stats.Stat +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.UserActionsFeature +import com.twitter.inject.utils.RetryUtils +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.util.MemoizingStatsReceiver +import com.twitter.stitch.Stitch +import com.twitter.util.Future +import com.twitter.util.Return +import com.twitter.util.Throw +import com.twitter.util.Try +import com.x.user_action_sequence.ActionName +import com.x.user_action_sequence.CandidateSet +import com.x.user_action_sequence.PredictNextActionsRequest +import com.x.user_action_sequence.PredictNextActionsResponse +import com.x.user_action_sequence.RecsysPredictorGrpc +import com.x.user_action_sequence.TweetBoolFeatures +import com.x.user_action_sequence.TweetInfo +import io.grpc.ManagedChannel +import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ +import scala.util.Random + +object PhoenixUtils { + private val TopLogProbsNum = 50 + private val MaxCandidates = 1400 + + def getTweetInfoFromCandidates( + candidateIds: Seq[TweetCandidate], + featureMaps: Seq[FeatureMap] + ): Seq[TweetInfo] = { + assert( + candidateIds.length == featureMaps.length, + "FeatureMap doesn't match candidateIds in length") + val candidateWithFeatureMaps = candidateIds.zip(featureMaps) + candidateWithFeatureMaps.take(MaxCandidates).map { + case (candidate, featureMap) => + val tweetBooleanFeatureBuilder = TweetBoolFeatures.newBuilder() + val (sourceTweetId, sourceAuthorId) = ( + featureMap.getOrElse(SourceTweetIdFeature, None), + featureMap.getOrElse(SourceUserIdFeature, None)) match { + case (Some(sourceTweetId), Some(sourceAuthorId)) => + tweetBooleanFeatureBuilder.setIsRetweet(true) + (sourceTweetId, sourceAuthorId) + case _ => + tweetBooleanFeatureBuilder.setIsRetweet(false) + (candidate.id, featureMap.get(AuthorIdFeature).get) + } + tweetBooleanFeatureBuilder + .setIsForYouPage(true) + .setIsPromotedTweet(false) + .setIsReply(featureMap.getOrElse(InReplyToTweetIdFeature, None).isDefined) + .setIsQuote(featureMap.getOrElse(QuotedTweetIdFeature, None).isDefined) + TweetInfo + .newBuilder() + .setTweetId(sourceTweetId) + .setAuthorId(sourceAuthorId) + .setTweetBoolFeatures(tweetBooleanFeatureBuilder.build()) + .build() + } + } + + def createCandidateSets( + query: PipelineQuery, + tweetInfos: Seq[TweetInfo] + ): PredictNextActionsRequest = { + val actionsSeqOpt = query.features.flatMap(_.getOrElse(UserActionsFeature, None)) + val candidateSets = CandidateSet + .newBuilder() + .setUserId(query.getRequiredUserId) + .addAllCandidates(tweetInfos.asJava) + .build() + + val predictionRequestBuilder = PredictNextActionsRequest + .newBuilder() + .addCandidateSets(candidateSets) + .setReturnLogprob(true) + .setTopLogprobsNum(TopLogProbsNum) + + actionsSeqOpt.foreach { actionsSeq => predictionRequestBuilder.addSequences(actionsSeq) } + predictionRequestBuilder.build() + } + + def predict( + request: PredictNextActionsRequest, + channels: Seq[ManagedChannel], + cluster: String, + timeoutMs: Int, + baseStat: MemoizingStatsReceiver, + ): Stitch[PredictNextActionsResponse] = { + + val timeStat = baseStat.scope("rpcTime") + val successStat = baseStat.scope("rpcSuccess") + val failureStat = baseStat.scope("rpcFailure") + val failureMsgStat = baseStat.scope("rpcFailureMsg") + + def attemptPredict(): Future[PredictNextActionsResponse] = { + // This is kept inside predict so that it doesn't use the same channel for retries + val channel = channels(Random.nextInt(channels.length)) + val recsysPredictorFutureStub = + RecsysPredictorGrpc + .newFutureStub(channel).withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS) + + FutureConverters + .RichListenableFuture(recsysPredictorFutureStub.predictNextActions(request)) + .toTwitter + } + + // Retry configuration: 2 attempts, 500ms per attempt, total 1s max + val retryPolicy = RetryPolicy + .tries[Try[PredictNextActionsResponse]](2, { case Throw(_) => true; case Return(_) => false }) + + val attemptPredictRetriedFuture = + RetryUtils.retryFuture[PredictNextActionsResponse](retryPolicy)(attemptPredict) + + Stitch + .callFuture(Stat.timeFuture(timeStat.stat(cluster))(attemptPredictRetriedFuture)) + .onFailure { e => + val errMsg = e.getMessage.replaceAll("\\W", "").takeRight(100) + failureStat.counter(cluster).incr() + failureMsgStat.scope(cluster).counter(errMsg).incr() + }.onSuccess { _ => + successStat.counter(cluster).incr() + } + } + + def getPredictionResponseMap( + request: PredictNextActionsRequest, + channels: Seq[ManagedChannel], + cluster: String, + timeoutMs: Int, + memoizingStatsReceiver: MemoizingStatsReceiver + ): Stitch[Map[Long, Map[ActionName, Double]]] = { + val predictionsResponse = predict(request, channels, cluster, timeoutMs, memoizingStatsReceiver) + predictionsResponse.map { predictions => + val distributionSetsList = predictions.getDistributionSetsList + val candidateDistributions = if (!distributionSetsList.isEmpty) { + distributionSetsList.get(0).getCandidateDistributionsList.asScala + } else Seq.empty + + candidateDistributions.map { distribution => + val candidateId = distribution.getCandidate.getTweetId + val probMap = distribution.getTopLogProbsList.asScala.zipWithIndex.map { + case (logProb, idx) => + ActionName.forNumber(idx) -> math.exp(logProb.doubleValue()) + }.toMap + candidateId -> probMap + }.toMap + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/RerankerUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/RerankerUtil.scala new file mode 100644 index 000000000..49ff468ad --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/RerankerUtil.scala @@ -0,0 +1,138 @@ +package com.twitter.home_mixer.util + +import com.twitter.finagle.stats.Counter +import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatures +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ConstantNegativeHead +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableNegSectionRankingParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NegativeScoreConstantFilterThresholdParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NegativeScoreNormFilterThresholdParam +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NormalizedNegativeHead +import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseWeightForNegHeadParam +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object RerankerUtil { + val Epsilon = 0.001 + + def computeModelScores( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate], + modelStatsOpt: Option[ModelStats] = None + ): Seq[(Double, Double)] = { + PredictedScoreFeatures.map { predictedScoreFeature => + val predictedScoreOpt = predictedScoreFeature.extractScore(candidate.features, query) + + modelStatsOpt.foreach(_.trackPredictedScoreStats(predictedScoreFeature, predictedScoreOpt)) + + val weight = + query.features.flatMap(_.get(predictedScoreFeature.weightQueryFeature)).getOrElse(0.0) + + val bias = predictedScoreFeature.biasQueryFeature + .flatMap(feature => query.features.flatMap(_.get(feature))).getOrElse(0.0) + + val score = + if (predictedScoreFeature.isEligible(candidate.features, query)) + predictedScoreOpt.getOrElse(0.0) + else bias + + (score, weight) + } + } + + def getScoresWithPerHeadMax( + scoresAndWeightsSeq: Seq[Seq[(Double, Double)]] + ): Seq[Seq[(Double, Double, Double)]] = { + if (scoresAndWeightsSeq.isEmpty) Seq.empty + else { + // Step 1: Transpose scores to group by heads + val headsScores: Seq[Seq[Double]] = scoresAndWeightsSeq.transpose.map { headScores => + headScores.map { case (score, _) => score } + } + + // Step 2: Get max scores per head + val headMaxScores: Seq[Double] = headsScores.map(_.max).toIndexedSeq + + // Step 3: Use max scores to get per head transformed scores + scoresAndWeightsSeq.map { candidateScores => + candidateScores.zipWithIndex.map { + case ((score, weight), headIdx) => + val headMaxScore = headMaxScores(headIdx) + (score, headMaxScore, weight) + } + } + } + } + + def computeDebugMetadata( + debugStr: String, + featureNames: Seq[String], + transformedScores: Seq[(Double, Double, Double)], + finalScore: Double + ): String = { + assert( + featureNames.size == transformedScores.size, + "Feature names size doesn't matter scores size") + val contributions: Seq[(String, Double)] = featureNames + .zip(transformedScores) + .collect { + case (feature, (score, _, weight)) if weight >= 0 => + (feature, score * weight) + } + + val topContributors: Seq[String] = contributions.collect { + case (name, contrib) if finalScore > 0 && (contrib / finalScore) > 0.3 => + f"$name:%%.2f".format(contrib / finalScore) + } + + debugStr + s" [${topContributors.mkString(", ")}]" + } + + def aggregateWeightedScores( + query: PipelineQuery, + scoresAndWeights: Seq[(Double, Double, Double)], + negativeFilterCounter: Counter + ): Double = { + val thresholdNegative = query.params(NegativeScoreConstantFilterThresholdParam) + val thresholdNegativeNormalized = query.params(NegativeScoreNormFilterThresholdParam) + val enableNegNormalized = query.params(NormalizedNegativeHead) + val enableNegConstant = query.params(ConstantNegativeHead) + val useWeightForNeg = query.params(UseWeightForNegHeadParam) + val negSectionRanking = query.params(EnableNegSectionRankingParam) + val (_, maxHeadScores, modelWeights) = scoresAndWeights.unzip3 + val combinedScoreSum: Double = { + scoresAndWeights.foldLeft(0.0) { + case (combinedScore, (score, maxHeadScore, weight)) => + if (weight >= 0.0 || useWeightForNeg) { + combinedScore + score * weight + } else { + // Apply filtering logic only for negative weights + val normScore = if (maxHeadScore == 0.0) 0.0 else score / maxHeadScore + val negFilterNorm = enableNegNormalized && normScore > thresholdNegativeNormalized + val negFilterConstant = enableNegConstant && score > thresholdNegative + if (negFilterNorm || negFilterConstant) { + negativeFilterCounter.incr() + // This should be shipped and cleaned as soon as possible + if (negSectionRanking) { + // Assumes negative scores are not greater than 0.9 and clipped to 1 + combinedScore + weight * (1.0 min (score + 0.1)) + } else + combinedScore + weight + } else combinedScore + } + } + } + + val positiveModelWeightsSum = modelWeights.filter(_ > 0.0).sum + val negativeModelWeightsSum = modelWeights.filter(_ < 0).sum.abs + val modelWeightsSum = positiveModelWeightsSum + negativeModelWeightsSum + + val weightedScoresSum = + if (modelWeightsSum == 0) combinedScoreSum.max(0.0) + else if (combinedScoreSum < 0) + (combinedScoreSum + negativeModelWeightsSum) / modelWeightsSum * Epsilon + else combinedScoreSum + Epsilon + + weightedScoresSum + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/SignalUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/SignalUtil.scala new file mode 100644 index 000000000..43f92b8e7 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/SignalUtil.scala @@ -0,0 +1,26 @@ +package com.twitter.home_mixer.util + +import com.twitter.home_mixer.model.HomeFeatures.LowSignalUserFeature +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.usersignalservice.{thriftscala => uss} + +object SignalUtil { + + val ExplicitSignals: Seq[uss.SignalType] = Seq( + uss.SignalType.TweetFavorite, + uss.SignalType.Retweet, + uss.SignalType.Reply, + uss.SignalType.TweetBookmarkV1, + uss.SignalType.TweetShareV1 + ) + + private val SmallFollowGraphSize = 5 + + def isLowSignalUser(query: PipelineQuery): Boolean = { + val followGraphSize = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty).size) + val smallFollowGraph = followGraphSize.exists(_ < SmallFollowGraphSize) + val lowSignal = query.features.map(_.getOrElse(LowSignalUserFeature, false)).getOrElse(false) + lowSignal && smallFollowGraph + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TensorFlowUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TensorFlowUtil.scala index 05d7c2127..14055273f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TensorFlowUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TensorFlowUtil.scala @@ -29,4 +29,10 @@ object TensorFlowUtil { val bb_content = skipEmbeddingBBHeader(bb) FloatTensor(byteBufferToFloatIterator(bb_content).map(_.toDouble).toList) } + + def embeddingNoHeaderByteBufferToFloatTensor( + bb: ByteBuffer + ): FloatTensor = { + FloatTensor(byteBufferToFloatIterator(bb).map(_.toDouble).toList) + } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/UrtUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/UrtUtil.scala new file mode 100644 index 000000000..aa0e47e47 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/UrtUtil.scala @@ -0,0 +1,33 @@ +package com.twitter.home_mixer.util + +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DeepLink +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ExternalUrl +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.UrtEndpoint +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.UrtEndpointOptions +import com.twitter.timelines.render.{thriftscala => t} + +object UrtUtil { + def transformUrl(url: t.Url): Url = { + val endpointOptions = url.urtEndpointOptions.map { options => + UrtEndpointOptions( + requestParams = options.requestParams.map(_.toMap), + title = options.title, + cacheId = options.cacheId, + subtitle = options.subtitle + ) + } + + val urlType = url.urlType match { + case t.UrlType.ExternalUrl => ExternalUrl + case t.UrlType.DeepLink => DeepLink + case t.UrlType.UrtEndpoint => UrtEndpoint + case t.UrlType.EnumUnknownUrlType(field) => + throw new UnknownUrlTypeException(field) + } + + Url(urlType = urlType, url = url.url, urtEndpointOptions = endpointOptions) + } +} +class UnknownUrlTypeException(field: Int) + extends UnsupportedOperationException(s"Unknown url type: $field") diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel index 2b5722179..b70674d5f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel @@ -4,16 +4,12 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "src/java/com/twitter/search/common/schema/base", "src/java/com/twitter/search/common/schema/earlybird", "src/java/com/twitter/search/common/util/lang", "src/java/com/twitter/search/queryparser/query:core-query-nodes", - "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-scala", - "src/thrift/com/twitter/search/common:query-scala", - "src/thrift/com/twitter/search/common:ranking-scala", "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala index 4c482b251..7f7a17588 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala @@ -16,6 +16,7 @@ object EarlybirdRequestUtil { val DefaultMaxHitsToProcess = 1000 val DefaultSearchProcessingTimeout: Duration = 200.milliseconds + val DefaultFeatureHydrationTimeout: Duration = 50.milliseconds val DefaultHydrationMaxNumResultsPerShard = 1000 val DefaultQueryMaxNumResultsPerShard = 300 val DefaultHydrationCollectorParams = mkCollectorParams(DefaultHydrationMaxNumResultsPerShard) @@ -77,7 +78,7 @@ object EarlybirdRequestUtil { userId: Option[Long], clientId: Option[String], skipVeryRecentTweets: Boolean, - followedUserIds: Set[Long], + queryUserIds: Set[Long], retweetsMutedUserIds: Set[Long], beforeTweetIdExclusive: Option[Long], afterTweetIdExclusive: Option[Long], @@ -87,11 +88,15 @@ object EarlybirdRequestUtil { authorScoreMap: Option[Map[Long, Double]] = None, tensorflowModel: Option[String] = None, ebModels: Seq[EarlybirdScoringModelConfig] = Seq.empty, - queryMaxNumResultsPerShard: Int = DefaultQueryMaxNumResultsPerShard + queryMaxNumResultsPerShard: Int = DefaultQueryMaxNumResultsPerShard, + enableExcludeSourceTweetIdsQuery: Boolean = false, + isVideoOnlyRequest: Boolean = false, + isRecency: Boolean = false, + getOlderTweets: Boolean = false, ): eb.EarlybirdRequest = { val QueryWithNamedDisjunctions(query, namedDisjunctionMap) = queryBuilder.create( - followedUserIds, + queryUserIds, retweetsMutedUserIds, beforeTweetIdExclusive, afterTweetIdExclusive, @@ -101,28 +106,45 @@ object EarlybirdRequestUtil { searchOperator = SearchOperator.Exclude, tweetFeatures = TweetFeatures.All, excludedTweetIds = excludedTweetIds.getOrElse(Set.empty), - enableExcludeSourceTweetIdsQuery = false + enableExcludeSourceTweetIdsQuery = enableExcludeSourceTweetIdsQuery, + isVideoOnlyRequest = isVideoOnlyRequest ) val ebRankingParams = getRankingParams(authorScoreMap, tensorflowModel, ebModels) val relOptions = RelevanceSearchUtil.RelevanceOptions.copy( - rankingParams = ebRankingParams + rankingParams = ebRankingParams, + returnAllResults = Some(false) ) - val followedUserIdsSeq = followedUserIds.toSeq + val metadataOptions = + if (isRecency) + eb.ThriftSearchResultMetadataOptions( + getTweetUrls = false, + getResultLocation = false, + deprecatedGetTopicIDs = false, + getInReplyToStatusId = true, + getReferencedTweetAuthorId = true, + getFromUserId = true + ) + else RelevanceSearchUtil.MetadataOptions + + val queryUserIdsSeq = queryUserIds.toSeq val namedDisjunctionMapOpt = if (namedDisjunctionMap.isEmpty) None else Some(namedDisjunctionMap.mapValues(_.toSeq)) + val rankingMode = + if (isRecency) eb.ThriftSearchRankingMode.Recency else eb.ThriftSearchRankingMode.Relevance + val thriftQuery = eb.ThriftSearchQuery( serializedQuery = Some(query.serialize), - fromUserIDFilter64 = Some(followedUserIdsSeq), + fromUserIDFilter64 = Some(queryUserIdsSeq), numResults = maxCount, collectConversationId = true, - rankingMode = eb.ThriftSearchRankingMode.Relevance, + rankingMode = rankingMode, relevanceOptions = Some(relOptions), collectorParams = Some(mkCollectorParams(queryMaxNumResultsPerShard)), facetFieldNames = Some(RelevanceSearchUtil.FacetsToFetch), - resultMetadataOptions = Some(RelevanceSearchUtil.MetadataOptions), + resultMetadataOptions = Some(metadataOptions), searcherId = userId, searchStatusIds = None, namedDisjunctionMap = namedDisjunctionMapOpt @@ -131,8 +153,8 @@ object EarlybirdRequestUtil { eb.EarlybirdRequest( searchQuery = thriftQuery, clientId = clientId, - getOlderResults = Some(false), - followedUserIds = Some(followedUserIdsSeq), + getOlderResults = Some(getOlderTweets), + followedUserIds = Some(queryUserIdsSeq), getProtectedTweetsOnly = Some(false), timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt, skipVeryRecentTweets = skipVeryRecentTweets, @@ -176,7 +198,8 @@ object EarlybirdRequestUtil { skipVeryRecentTweets = true, // This param decides # of tweets to return from search superRoot and realtime/protected/Archive roots. // It takes higher precedence than ThriftSearchQuery.numResults - numResultsToReturnAtRoot = Some(candidateSize) + numResultsToReturnAtRoot = Some(candidateSize), + partitionTimeoutMs = Some(DefaultFeatureHydrationTimeout.inMillis.toInt) ) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala index 06e0dd708..268f918b8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala @@ -1,5 +1,6 @@ package com.twitter.home_mixer.util.earlybird +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.search.common.constants.{thriftscala => scc} import com.twitter.search.common.features.{thriftscala => sc} import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant @@ -7,30 +8,32 @@ import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.Earlyb import com.twitter.search.common.util.lang.ThriftLanguageUtil import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.timelines.earlybird.common.utils.InNetworkEngagement +import com.twitter.util.Future object EarlybirdResponseUtil { private[earlybird] val Mentions: String = "mentions" - private[earlybird] val Hashtags: String = "hashtags" private val CharsToRemoveFromMentions: Set[Char] = "@".toSet - private val CharsToRemoveFromHashtags: Set[Char] = "#".toSet // Default value of settings of ThriftTweetFeatures. private[earlybird] val DefaultEarlybirdFeatures: sc.ThriftTweetFeatures = sc.ThriftTweetFeatures() private[earlybird] val DefaultCount = 0 private[earlybird] val DefaultLanguage = 0 private[earlybird] val DefaultScore = 0.0 + private[earlybird] val DefaultEBResponseProcessParallelism = 32 - private[earlybird] def getTweetCountByAuthorId( + def getTweetCountByAuthorId( searchResults: Seq[eb.ThriftSearchResult] ): Map[Long, Int] = { - searchResults - .groupBy { result => - result.metadata.map(_.fromUserId).getOrElse(0L) - }.mapValues(_.size).withDefaultValue(0) + val tweetCounts = scala.collection.mutable.Map.empty[Long, Int] + searchResults.foreach { result => + val authorId = result.metadata.map(_.fromUserId).getOrElse(0L) + tweetCounts(authorId) = tweetCounts.getOrElse(authorId, 0) + 1 + } + tweetCounts.toMap.withDefaultValue(0) } - private[earlybird] def getLanguage(uiLanguageCode: Option[String]): Option[scc.ThriftLanguage] = { + def getLanguage(uiLanguageCode: Option[String]): Option[scc.ThriftLanguage] = { uiLanguageCode.flatMap { languageCode => scc.ThriftLanguage.get(ThriftLanguageUtil.getThriftLanguageOf(languageCode).getValue) } @@ -41,11 +44,6 @@ object EarlybirdResponseUtil { getFacets(facetLabels, Mentions, CharsToRemoveFromMentions) } - private def getHashtags(result: eb.ThriftSearchResult): Seq[String] = { - val facetLabels = result.metadata.flatMap(_.facetLabels).getOrElse(Seq.empty) - getFacets(facetLabels, Hashtags, CharsToRemoveFromHashtags) - } - private def getFacets( facetLabels: Seq[eb.ThriftFacetLabel], facetName: String, @@ -136,33 +134,41 @@ object EarlybirdResponseUtil { uiLanguageCode: Option[String] = None, followedUserIds: Set[Long], mutuallyFollowingUserIds: Set[Long], - searchResults: Seq[eb.ThriftSearchResult], - sourceTweetSearchResults: Seq[eb.ThriftSearchResult], - ): Map[Long, sc.ThriftTweetFeatures] = { + idToSearchResults: Map[Long, eb.ThriftSearchResult] + ): Future[Map[Long, sc.ThriftTweetFeatures]] = { - val allSearchResults = searchResults ++ sourceTweetSearchResults - val sourceTweetSearchResultById = - sourceTweetSearchResults.map(result => (result.id -> result)).toMap + val searchResults = idToSearchResults.values.toSeq val inNetworkEngagement = - InNetworkEngagement(followedUserIds.toSeq, mutuallyFollowingUserIds, allSearchResults) - searchResults.map { searchResult => - val features = getThriftTweetFeaturesFromSearchResult( - searcherUserId, - screenName, - userLanguages, - getLanguage(uiLanguageCode), - getTweetCountByAuthorId(searchResults), - followedUserIds, - mutuallyFollowingUserIds, - sourceTweetSearchResultById, - inNetworkEngagement, - searchResult - ) - (searchResult.id -> features) - }.toMap + InNetworkEngagement(followedUserIds.toSeq, mutuallyFollowingUserIds, searchResults) + val tweetCountByAuthorId = getTweetCountByAuthorId(searchResults) + val uiLanguage = getLanguage(uiLanguageCode) + val idWithFeaturesSeqFu = OffloadFuturePools.parallelize[ + eb.ThriftSearchResult, + (Long, sc.ThriftTweetFeatures) + ]( + inputSeq = searchResults, + transformer = (searchResult: eb.ThriftSearchResult) => + ( + searchResult.id, + getThriftTweetFeaturesFromSearchResult( + searcherUserId, + screenName, + userLanguages, + uiLanguage, + tweetCountByAuthorId, + followedUserIds, + mutuallyFollowingUserIds, + idToSearchResults, + inNetworkEngagement, + searchResult + )), + parallelism = DefaultEBResponseProcessParallelism, + ) + idWithFeaturesSeqFu.map(idWithFeaturesSeq => + idWithFeaturesSeq.map(idWithFeatures => idWithFeatures._1 -> idWithFeatures._2).toMap) } - private[earlybird] def getThriftTweetFeaturesFromSearchResult( + def getThriftTweetFeaturesFromSearchResult( searcherUserId: Long, screenName: Option[String], userLanguages: Seq[scc.ThriftLanguage], @@ -170,7 +176,7 @@ object EarlybirdResponseUtil { tweetCountByAuthorId: Map[Long, Int], followedUserIds: Set[Long], mutuallyFollowingUserIds: Set[Long], - sourceTweetSearchResultById: Map[Long, eb.ThriftSearchResult], + idToSearchResults: Map[Long, eb.ThriftSearchResult], inNetworkEngagement: InNetworkEngagement, searchResult: eb.ThriftSearchResult, ): sc.ThriftTweetFeatures = { @@ -185,7 +191,7 @@ object EarlybirdResponseUtil { tweetCountByAuthorId, followedUserIds, mutuallyFollowingUserIds, - sourceTweetSearchResultById, + idToSearchResults, inNetworkEngagement, searchResult )(_) @@ -205,10 +211,6 @@ object EarlybirdResponseUtil { val isRetweet = metadata.isRetweet.getOrElse(false) val isReply = metadata.isReply.getOrElse(false) - // Facets. - val mentions = getMentions(result) - val hashtags = getHashtags(result) - val searchResultSchemaFeatures = metadata.extraMetadata.flatMap(_.features) val booleanSearchResultSchemaFeatures = searchResultSchemaFeatures.flatMap(_.boolValues) val intSearchResultSchemaFeatures = searchResultSchemaFeatures.flatMap(_.intValues) @@ -337,8 +339,6 @@ object EarlybirdResponseUtil { lastQuoteSinceCreationHrs = getIntOptFeature(LAST_QUOTE_SINCE_CREATION_HRS, intSearchResultSchemaFeatures), likedByUserIds = metadata.extraMetadata.flatMap(_.likedByUserIds), - mentionsList = if (mentions.nonEmpty) Some(mentions) else None, - hashtagsList = if (hashtags.nonEmpty) Some(hashtags) else None, isComposerSourceCamera = getBooleanOptFeature(COMPOSER_SOURCE_IS_CAMERA_FLAG, booleanSearchResultSchemaFeatures), ) @@ -356,7 +356,7 @@ object EarlybirdResponseUtil { tweetCountByAuthorId: Map[Long, Int], followedUserIds: Set[Long], mutuallyFollowingUserIds: Set[Long], - sourceTweetSearchResultById: Map[Long, eb.ThriftSearchResult], + idToSearchResults: Map[Long, eb.ThriftSearchResult], inNetworkEngagement: InNetworkEngagement, result: eb.ThriftSearchResult )( @@ -366,7 +366,7 @@ object EarlybirdResponseUtil { .map { metadata => val isRetweet = metadata.isRetweet.getOrElse(false) val sourceTweet = - if (isRetweet) sourceTweetSearchResultById.get(metadata.sharedStatusId) + if (isRetweet) idToSearchResults.get(metadata.sharedStatusId) else None val mentionsInSourceTweet = sourceTweet.map(getMentions).getOrElse(Seq.empty) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala index 40b727141..ce283cd3f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala @@ -6,8 +6,8 @@ import com.twitter.search.earlybird.{thriftscala => eb} object RelevanceSearchUtil { val Mentions: String = EarlybirdFieldConstant.MENTIONS_FACET - val Hashtags: String = EarlybirdFieldConstant.HASHTAGS_FACET - val FacetsToFetch: Seq[String] = Seq(Mentions, Hashtags) + val FacetsToFetch: Seq[String] = Seq(Mentions) + val MaxHitsToProcess: Int = 1500 val MetadataOptions: eb.ThriftSearchResultMetadataOptions = { eb.ThriftSearchResultMetadataOptions( @@ -19,7 +19,6 @@ object RelevanceSearchUtil { getMediaBits = true, getAllFeatures = true, returnSearchResultFeatures = true, - // Set getExclusiveConversationAuthorId in order to retrieve Exclusive / SuperFollow tweets. getExclusiveConversationAuthorId = true ) } @@ -29,10 +28,11 @@ object RelevanceSearchUtil { proximityScoring = true, maxConsecutiveSameUser = Some(2), rankingParams = None, - maxHitsToProcess = Some(500), + maxHitsToProcess = Some(MaxHitsToProcess), maxUserBlendCount = Some(3), proximityPhraseWeight = 9.0, returnAllResults = Some(true) ) } + } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/BUILD.bazel index 5acfd98e0..a4a5f96b7 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/BUILD.bazel @@ -5,15 +5,5 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "src/java/com/twitter/common/text/tagger", - "src/java/com/twitter/common/text/token", - "src/java/com/twitter/common_internal/text", - "src/java/com/twitter/common_internal/text/version", - "src/java/com/twitter/search/common/util/text", - "src/thrift/com/twitter/search/common:features-scala", - "src/thrift/com/twitter/tweetypie:media-entity-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/FeatureExtractionHelper.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/FeatureExtractionHelper.scala index 07cbdebe4..9ce311211 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/FeatureExtractionHelper.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/FeatureExtractionHelper.scala @@ -6,7 +6,8 @@ import com.twitter.tweetypie.{thriftscala => tp} object FeatureExtractionHelper { def extractFeatures( - tweet: tp.Tweet + tweet: tp.Tweet, + isExtractMediaEntities: Boolean = true ): ContentFeatures = { val contentFeaturesFromTweet = ContentFeatures.Empty.copy( selfThreadMetadata = tweet.selfThreadMetadata @@ -18,7 +19,8 @@ object FeatureExtractionHelper { ) val contentFeaturesWithMedia = TweetMediaFeaturesExtractor.addMediaFeaturesFromTweet( contentFeaturesWithText, - tweet + tweet, + isExtractMediaEntities ) contentFeaturesWithMedia.copy( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetMediaFeaturesExtractor.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetMediaFeaturesExtractor.scala index 0a5a93a2e..1fe1ed5b8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetMediaFeaturesExtractor.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetMediaFeaturesExtractor.scala @@ -32,72 +32,106 @@ object TweetMediaFeaturesExtractor { } } + def getMediaIds(tweet: tp.Tweet): Seq[Long] = { + val tweetMediaIdsOpt = tweet.media.map { mediaEntities => + mediaEntities.map { mediaEntity => + mediaEntity.mediaId + } + } + tweetMediaIdsOpt.getOrElse(Seq.empty) + } + def addMediaFeaturesFromTweet( inputFeatures: ContentFeatures, tweet: tp.Tweet, + isExtractMediaEntities: Boolean = true ): ContentFeatures = { - val featuresWithMediaEntity = tweet.media - .map { mediaEntities => - val sizeFeatures = getSizeFeatures(mediaEntities) - val playbackFeatures = getPlaybackFeatures(mediaEntities) - val mediaWidths = sizeFeatures.map(_.width.toShort) - val mediaHeights = sizeFeatures.map(_.height.toShort) - val resizeMethods = sizeFeatures.map(_.resizeMethod.toShort) - val faceMapAreas = getFaceMapAreas(mediaEntities) - val sortedColorPalette = getSortedColorPalette(mediaEntities) - val stickerFeatures = getStickerFeatures(mediaEntities) - val mediaOriginProviders = getMediaOriginProviders(mediaEntities) - val isManaged = getIsManaged(mediaEntities) - val is360 = getIs360(mediaEntities) - val viewCount = getViewCount(mediaEntities) - val userDefinedProductMetadataFeatures = - getUserDefinedProductMetadataFeatures(mediaEntities) - val isMonetizable = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.isMonetizable)) - val isEmbeddable = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.isEmbeddable)) - val hasSelectedPreviewImage = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasSelectedPreviewImage)) - val hasTitle = getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasTitle)) - val hasDescription = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasDescription)) - val hasVisitSiteCallToAction = getOptBooleanFromSeqOpt( - userDefinedProductMetadataFeatures.map(_.hasVisitSiteCallToAction)) - val hasAppInstallCallToAction = getOptBooleanFromSeqOpt( - userDefinedProductMetadataFeatures.map(_.hasAppInstallCallToAction)) - val hasWatchNowCallToAction = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasWatchNowCallToAction)) + val featuresWithMediaEntity = + if (isExtractMediaEntities) { + tweet.media + .map { mediaEntities => + val sizeFeatures = getSizeFeatures(mediaEntities) + val playbackFeatures = getPlaybackFeatures(mediaEntities) + val mediaWidths = sizeFeatures.map(_.width.toShort) + val mediaHeights = sizeFeatures.map(_.height.toShort) + val resizeMethods = sizeFeatures.map(_.resizeMethod.toShort) + val faceMapAreas = getFaceMapAreas(mediaEntities) + val sortedColorPalette = getSortedColorPalette(mediaEntities) + val stickerFeatures = getStickerFeatures(mediaEntities) + val mediaOriginProviders = getMediaOriginProviders(mediaEntities) + val isManaged = getIsManaged(mediaEntities) + val is360 = getIs360(mediaEntities) + val viewCount = getViewCount(mediaEntities) + val userDefinedProductMetadataFeatures = + getUserDefinedProductMetadataFeatures(mediaEntities) + val isMonetizable = + getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.isMonetizable)) + val isEmbeddable = + getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.isEmbeddable)) + val hasSelectedPreviewImage = + getOptBooleanFromSeqOpt( + userDefinedProductMetadataFeatures.map(_.hasSelectedPreviewImage)) - inputFeatures.copy( - videoDurationMs = playbackFeatures.durationMs, - bitRate = playbackFeatures.bitRate, - aspectRatioNum = playbackFeatures.aspectRatioNum, - aspectRatioDen = playbackFeatures.aspectRatioDen, - widths = Some(mediaWidths), - heights = Some(mediaHeights), - resizeMethods = Some(resizeMethods), - faceAreas = Some(faceMapAreas), - dominantColorRed = sortedColorPalette.headOption.map(_.rgb.red), - dominantColorBlue = sortedColorPalette.headOption.map(_.rgb.blue), - dominantColorGreen = sortedColorPalette.headOption.map(_.rgb.green), - dominantColorPercentage = sortedColorPalette.headOption.map(_.percentage), - numColors = Some(sortedColorPalette.size.toShort), - stickerIds = Some(stickerFeatures), - mediaOriginProviders = Some(mediaOriginProviders), - isManaged = Some(isManaged), - is360 = Some(is360), - viewCount = viewCount, - isMonetizable = isMonetizable, - isEmbeddable = isEmbeddable, - hasSelectedPreviewImage = hasSelectedPreviewImage, - hasTitle = hasTitle, - hasDescription = hasDescription, - hasVisitSiteCallToAction = hasVisitSiteCallToAction, - hasAppInstallCallToAction = hasAppInstallCallToAction, - hasWatchNowCallToAction = hasWatchNowCallToAction - ) - } - .getOrElse(inputFeatures) + val hasImage = Some( + mediaEntities.exists { entity => + entity.mediaKey.exists { key => + ImageCategories.contains(key.mediaCategory.value) + } + } + ) + val hasVideo = Some( + mediaEntities.exists { entity => + entity.mediaKey.exists { key => + VideoCategories.contains(key.mediaCategory.value) + } + } + ) + + val hasTitle = + getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasTitle)) + val hasDescription = + getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasDescription)) + val hasVisitSiteCallToAction = getOptBooleanFromSeqOpt( + userDefinedProductMetadataFeatures.map(_.hasVisitSiteCallToAction)) + val hasAppInstallCallToAction = getOptBooleanFromSeqOpt( + userDefinedProductMetadataFeatures.map(_.hasAppInstallCallToAction)) + val hasWatchNowCallToAction = + getOptBooleanFromSeqOpt( + userDefinedProductMetadataFeatures.map(_.hasWatchNowCallToAction)) + + inputFeatures.copy( + videoDurationMs = playbackFeatures.durationMs, + bitRate = playbackFeatures.bitRate, + aspectRatioNum = playbackFeatures.aspectRatioNum, + aspectRatioDen = playbackFeatures.aspectRatioDen, + widths = Some(mediaWidths), + heights = Some(mediaHeights), + resizeMethods = Some(resizeMethods), + faceAreas = Some(faceMapAreas), + dominantColorRed = sortedColorPalette.headOption.map(_.rgb.red), + dominantColorBlue = sortedColorPalette.headOption.map(_.rgb.blue), + dominantColorGreen = sortedColorPalette.headOption.map(_.rgb.green), + dominantColorPercentage = sortedColorPalette.headOption.map(_.percentage), + numColors = Some(sortedColorPalette.size.toShort), + stickerIds = Some(stickerFeatures), + mediaOriginProviders = Some(mediaOriginProviders), + isManaged = Some(isManaged), + is360 = Some(is360), + viewCount = viewCount, + isMonetizable = isMonetizable, + isEmbeddable = isEmbeddable, + hasSelectedPreviewImage = hasSelectedPreviewImage, + hasTitle = hasTitle, + hasDescription = hasDescription, + hasVisitSiteCallToAction = hasVisitSiteCallToAction, + hasAppInstallCallToAction = hasAppInstallCallToAction, + hasWatchNowCallToAction = hasWatchNowCallToAction, + hasImage = hasImage, + hasVideo = hasVideo + ) + } + .getOrElse(inputFeatures) + } else inputFeatures val featuresWithMediaTags = tweet.mediaTags .map { mediaTags => @@ -151,7 +185,7 @@ object TweetMediaFeaturesExtractor { case playbackFeatures: PlaybackFeatures => playbackFeatures } - if (allPlaybackFeatures.nonEmpty) allPlaybackFeatures.maxBy(_.durationMs) + if (allPlaybackFeatures.nonEmpty) allPlaybackFeatures.minBy(_.durationMs) else PlaybackFeatures(None, None, None, None) } diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/BUILD.bazel new file mode 100644 index 000000000..601f344e9 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/BUILD.bazel @@ -0,0 +1,26 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "finatra-internal/mtls-http/src/main/scala", + "geoduck/service/src/main/scala/com/twitter/geoduck/service/common/clientmodules", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", + "thrift-web-forms/src/main/scala/com/twitter/thriftwebforms/view", + "tweet-mixer/server/src/main/resources", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/controller", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/store", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], + exports = [ + "finatra/inject/inject-slf4j/src/main/scala/com/twitter/inject", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerHttpServerWarmupHandler.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerHttpServerWarmupHandler.scala new file mode 100644 index 000000000..16a38f815 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerHttpServerWarmupHandler.scala @@ -0,0 +1,20 @@ +package com.twitter.tweet_mixer + +import com.twitter.finatra.http.routing.HttpWarmup +import com.twitter.finatra.http.request.RequestBuilder._ +import com.twitter.inject.utils.Handler +import com.twitter.util.Try +import com.twitter.util.logging.Logging +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TweetMixerHttpServerWarmupHandler @Inject() (warmup: HttpWarmup) + extends Handler + with Logging { + + override def handle(): Unit = { + Try(warmup.send(get("/admin/product-mixer/product-pipelines"), admin = true)()) + .onFailure(e => error(e.getMessage, e)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerServer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerServer.scala new file mode 100644 index 000000000..6c9fa9f52 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerServer.scala @@ -0,0 +1,169 @@ +package com.twitter.tweet_mixer + +import com.google.inject.Module +import com.twitter.finagle.Filter +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.mux.transport.Compression +import com.twitter.finagle.mux.transport.CompressionLevel +import com.twitter.finatra.annotations.DarkTrafficFilterType +import com.twitter.finatra.http.HttpServer +import com.twitter.finatra.http.routing.HttpRouter +import com.twitter.finatra.mtls.http.{Mtls => HttpMtls} +import com.twitter.finatra.mtls.thriftmux.Mtls +import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule +import com.twitter.finatra.thrift.ThriftServer +import com.twitter.finatra.thrift.filters._ +import com.twitter.finatra.thrift.routing.ThriftRouter +import com.twitter.geoduck.service.common.clientmodules.GeoduckUserLocateModule +import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule +import com.twitter.product_mixer.component_library.module.FeedbackHistoryClientModule +import com.twitter.product_mixer.component_library.module.GizmoduckClientModule +import com.twitter.product_mixer.component_library.module.MemcachedImpressionBloomFilterStoreModule +import com.twitter.product_mixer.component_library.module.MemcachedVideoImpressionBloomFilterStoreModule +import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule +import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule +import com.twitter.product_mixer.component_library.module.TestUserMapperConfigModule +import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule +import com.twitter.product_mixer.component_library.module.UtegClientModule +import com.twitter.product_mixer.component_library.module.UserSessionStoreModule +import com.twitter.product_mixer.core.controllers.ProductMixerController +import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper +import com.twitter.product_mixer.core.module.ProductMixerModule +import com.twitter.product_mixer.core.module.StratoClientModule +import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule +import com.twitter.tweet_mixer.controller.TweetMixerThriftController +import com.twitter.tweet_mixer.module.CertoStratoTopicTweetsStoreModule +import com.twitter.tweet_mixer.module.ExtendedStratoClientModule +import com.twitter.tweet_mixer.module.GPURetrievalHttpClientModule +import com.twitter.tweet_mixer.module.HaploliteClientModule +import com.twitter.tweet_mixer.module.HydraEmbeddingGenerationServiceClientModule +import com.twitter.tweet_mixer.module.HydraRootClientModule +import com.twitter.tweet_mixer.module.InMemoryCacheModule +import com.twitter.tweet_mixer.module.MHMtlsParamsModule +import com.twitter.tweet_mixer.module.ManhattanFeatureRepositoryModule +import com.twitter.tweet_mixer.module.MemcacheClientModule +import com.twitter.tweet_mixer.module.PipelineFailureExceptionMapper +import com.twitter.tweet_mixer.module.SampleFeatureStoreV1DynamicClientBuilderModule +import com.twitter.tweet_mixer.module.SimClustersANNServiceNameToClientMapper +import com.twitter.tweet_mixer.module.SkitStratoTopicTweetsStoreModule +import com.twitter.tweet_mixer.module.StitchMemcacheClientModule +import com.twitter.tweet_mixer.module.TimeoutConfigModule +import com.twitter.tweet_mixer.module.TwHINANNServiceModule +import com.twitter.tweet_mixer.module.TwHINEmbeddingStoreModule +import com.twitter.tweet_mixer.module.TweetMixerFlagModule +import com.twitter.tweet_mixer.module.UserStateStoreModule +import com.twitter.tweet_mixer.module.thrift_client.AnnEmbeddingProducerModule +import com.twitter.tweet_mixer.module.thrift_client.AnnQueryServiceClientModule +import com.twitter.tweet_mixer.module.thrift_client.AnnQueryableByIdModule +import com.twitter.tweet_mixer.module.thrift_client.EarlybirdRealtimeCGModule +import com.twitter.tweet_mixer.module.thrift_client.GeoduckHydrationClientModule +import com.twitter.tweet_mixer.module.thrift_client.GeoduckLocationServiceClientModule +import com.twitter.tweet_mixer.module.thrift_client.SimClustersAnnServiceClientModule +import com.twitter.tweet_mixer.module.thrift_client.TweetyPieClientModule +import com.twitter.tweet_mixer.module.thrift_client.UserTweetGraphClientModule +import com.twitter.tweet_mixer.module.thrift_client.UserVideoGraphClientModule +import com.twitter.tweet_mixer.module.thrift_client.VecDBAnnServiceClientModule +import com.twitter.tweet_mixer.param.GlobalParamConfigModule +import com.twitter.tweet_mixer.product.TweetMixerProductModule +import com.twitter.tweet_mixer.{thriftscala => t} +import com.twitter.tweet_mixer.module.thrift_client.QigServiceClientModule + +object TweetMixerServerMain extends TweetMixerServer + +class TweetMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls { + override val name = "tweet-mixer-server" + + override val modules: Seq[Module] = Seq( + AnnEmbeddingProducerModule, + AnnQueryableByIdModule, + AnnQueryServiceClientModule, + CertoStratoTopicTweetsStoreModule, + new DarkTrafficFilterModule[t.TweetMixer.ReqRepServicePerEndpoint](), + EarlybirdRealtimeCGModule, + FeedbackHistoryClientModule, + GeoduckHydrationClientModule, + GeoduckLocationServiceClientModule, + GeoduckUserLocateModule, + GizmoduckClientModule, + GlobalParamConfigModule, + GPURetrievalHttpClientModule, + HaploliteClientModule, + HydraRootClientModule, + HydraEmbeddingGenerationServiceClientModule, + InMemoryCacheModule, + ManhattanFeatureRepositoryModule, + MemcacheClientModule, + MemcachedImpressionBloomFilterStoreModule, + MemcachedVideoImpressionBloomFilterStoreModule, + MHMtlsParamsModule, + new MtlsThriftWebFormsModule[t.TweetMixer.MethodPerEndpoint](this), + OnboardingTaskServiceModule, + ProductMixerModule, + new ProductScopeStringCenterModule(), + QigServiceClientModule, + SampleFeatureStoreV1DynamicClientBuilderModule, + SimClustersAnnServiceClientModule, + SimClustersANNServiceNameToClientMapper, + SkitStratoTopicTweetsStoreModule, + SocialGraphServiceModule, + StitchMemcacheClientModule, + StratoClientModule, + TestUserMapperConfigModule, + TimeoutConfigModule, + TweetMixerFlagModule, + TweetMixerProductModule, + TweetyPieClientModule, + TwHINANNServiceModule, + TimelineServiceClientModule, + TwHINEmbeddingStoreModule, + UserTweetGraphClientModule, + UserVideoGraphClientModule, + UserStateStoreModule, + UtegClientModule, + VecDBAnnServiceClientModule, + ExtendedStratoClientModule, + UserSessionStoreModule + ) + + def configureThrift(router: ThriftRouter): Unit = { + router + .filter[LoggingMDCFilter] + .filter[TraceIdMDCFilter] + .filter[ThriftMDCFilter] + .filter[StatsFilter] + .filter[AccessLoggingFilter] + .filter[ExceptionMappingFilter] + .filter[Filter.TypeAgnostic, DarkTrafficFilterType] + .exceptionMapper[LoggingThrowableExceptionMapper] + .exceptionMapper[PipelineFailureExceptionMapper] + .add[TweetMixerThriftController] + } + + override def configureHttp(router: HttpRouter): Unit = { + router.add( + ProductMixerController[t.TweetMixer.MethodPerEndpoint]( + this.injector, + t.TweetMixer.ExecutePipeline + ) + ) + } + + override def configureThriftServer(server: ThriftMux.Server): ThriftMux.Server = { + val compressor = Seq(Compression.lz4Compressor(highCompression = true)) + val decompressor = Seq(Compression.lz4Decompressor()) + + // Mixer services accept compression to speed up debug requests from Turntable (which requests + // compression). Normal service calls will remain uncompressed unless the requesting service + // explicitly asks for compression. + val compressionLevel = CompressionLevel.Accepted + + server.withCompressionPreferences + .compression(compressionLevel, compressor) + .withCompressionPreferences.decompression(compressionLevel, decompressor) + } + + override protected def warmup(): Unit = { + handle[TweetMixerThriftServerWarmupHandler]() + handle[TweetMixerHttpServerWarmupHandler]() + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerThriftServerWarmupHandler.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerThriftServerWarmupHandler.scala new file mode 100644 index 000000000..151b1e98c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/TweetMixerThriftServerWarmupHandler.scala @@ -0,0 +1,77 @@ +package com.twitter.tweet_mixer + +import com.twitter.finagle.thrift.ClientId +import com.twitter.finatra.thrift.routing.ThriftWarmup +import com.twitter.inject.utils.Handler +import com.twitter.product_mixer.core.{thriftscala => pt} +import com.twitter.tweet_mixer.{thriftscala => st} +import com.twitter.scrooge.Request +import com.twitter.scrooge.Response +import com.twitter.util.Return +import com.twitter.util.Throw +import com.twitter.util.Try +import com.twitter.util.logging.Logging +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TweetMixerThriftServerWarmupHandler @Inject() (warmup: ThriftWarmup) + extends Handler + with Logging { + + private val clientId = ClientId("thrift-warmup-client") + + def handle(): Unit = { + val testIds = Seq(1, 2, 3) + try { + clientId.asCurrent { + testIds.foreach { id => + val warmupReq = warmupQuery(id) + info(s"Sending warm-up request to service with query: $warmupReq") + warmup.sendRequest( + method = st.TweetMixer.GetRecommendationResponse, + req = Request(st.TweetMixer.GetRecommendationResponse.Args(warmupReq)))( + assertWarmupResponse) + } + } + } catch { + case e: Throwable => + error(e.getMessage, e) + } + info("Warm-up done.") + } + + private def warmupQuery(userId: Long): st.TweetMixerRequest = { + val clientContext = pt.ClientContext( + userId = Some(userId), + guestId = None, + appId = Some(1L), + ipAddress = Some("0.0.0.0"), + userAgent = Some("FAKE_USER_AGENT_FOR_WARMUPS"), + countryCode = Some("US"), + languageCode = Some("en"), + isTwoffice = None, + userRoles = None, + deviceId = Some("FAKE_DEVICE_ID_FOR_WARMUPS") + ) + st.TweetMixerRequest( + clientContext = clientContext, + product = st.Product.HomeRecommendedTweets, + productContext = Some( + st.ProductContext.HomeRecommendedTweetsProductContext( + st.HomeRecommendedTweetsProductContext())), + maxResults = Some(3) + ) + } + + private def assertWarmupResponse( + result: Try[Response[st.TweetMixer.GetRecommendationResponse.SuccessType]] + ): Unit = { + result match { + case Return(_) => // ok + case Throw(exception) => + warn("Error performing warm-up request.") + error(exception.getMessage, exception) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/BUILD.bazel new file mode 100644 index 000000000..6a1aa17a2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/BUILD.bazel @@ -0,0 +1,61 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "ann/src/main/scala/com/twitter/ann/hnsw", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/uteg", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "src/scala/com/twitter/simclusters_v2/common", + "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", + "src/thrift/com/twitter/search:earlybird-scala", + "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", + "strato/config/columns/recommendations/simclusters_v2:simclusters_v2-strato-client", + "strato/config/columns/timelines/local:local-strato-client", + "strato/config/columns/trendsai/media:media-strato-client", + "timelines/src/main/scala/com/twitter/timelines/clients/haplolite", + "timelineservice/common:model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/content_embedding_ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/curated_user_tls_per_language", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/events", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/user_location", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], + exports = [ + "strato/config/columns/recommendations/simclusters_v2:simclusters_v2-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/events", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/CertoTopicTweetsCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/CertoTopicTweetsCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..9b4feb5f4 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/CertoTopicTweetsCandidatePipelineConfigFactory.scala @@ -0,0 +1,93 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.candidate_source.topic_tweets.CertoTopicTweetsCandidateSource +import com.twitter.tweet_mixer.candidate_source.topic_tweets.CertoTopicTweetsQuery +import com.twitter.tweet_mixer.functional_component.gate.AllowLowSignalUserGate +import com.twitter.tweet_mixer.functional_component.transformer.TopicTweetFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.CertoTopicTweetsQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.CertoTopicTweetsParams +import com.twitter.tweet_mixer.param.CertoTopicTweetsParams.CertoTopicTweetsEnable +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableUserTopicIdsHydrator +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class CertoTopicTweetsCandidatePipelineConfigFactory @Inject() ( + certoTopicTweetsCandidateSource: CertoTopicTweetsCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + topicsSignalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): CertoTopicTweetsCandidatePipelineConfig[Query] = { + new CertoTopicTweetsCandidatePipelineConfig( + identifierPrefix = identifierPrefix, + topicsSignalFn = topicsSignalFn, + certoTopicTweetsCandidateSource = certoTopicTweetsCandidateSource + ) + } +} + +class CertoTopicTweetsCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + topicsSignalFn: PipelineQuery => Seq[Long], + certoTopicTweetsCandidateSource: CertoTopicTweetsCandidateSource +)( + implicit val notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + CertoTopicTweetsQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.CertoTopicTweets) + + override val supportedClientParam: Option[FSParam[Boolean]] = Some(CertoTopicTweetsEnable) + + override val gates: Seq[Gate[PipelineQuery]] = Seq(AllowLowSignalUserGate) + + override val queryTransformer: CandidatePipelineQueryTransformer[Query, CertoTopicTweetsQuery] = + CertoTopicTweetsQueryTransformer( + maxTweetsPerTopicParam = CertoTopicTweetsParams.MaxNumCandidatesPerTopic, + maxTweetsParam = CertoTopicTweetsParams.MaxNumCandidates, + minCertoScoreParam = CertoTopicTweetsParams.MinCertoScore, + minCertoFavCountParam = CertoTopicTweetsParams.MinFavCount, + userInferredTopicIdsEnabled = EnableUserTopicIdsHydrator, + signalsFn = query => topicsSignalFn(query) + ) + + override def candidateSource: CandidateSource[ + CertoTopicTweetsQuery, + TweetMixerCandidate + ] = certoTopicTweetsCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TopicTweetFeatureTransformer) + + override val alerts = Seq( + defaultSuccessRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentAnnTweetBasedCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentAnnTweetBasedCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..c3c1a97d3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentAnnTweetBasedCandidatePipelineConfigFactory.scala @@ -0,0 +1,139 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.candidate_source.content_embedding_ann.ContentEmbeddingAnnCandidateSource +import com.twitter.tweet_mixer.candidate_source.content_embedding_ann.ContentEmbeddingAnnQuery +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.ContentEmbeddingAnnParams +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableContentEmbeddingAnnTweets +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.BusinessHours +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ContentAnnTweetBasedCandidatePipelineConfigFactory @Inject() ( + contentEmbeddingAnnCandidateSource: ContentEmbeddingAnnCandidateSource) { + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long], + ): ContentAnnTweetBasedCandidatePipelineConfig[Query] = { + new ContentAnnTweetBasedCandidatePipelineConfig( + identifierPrefix, + signalsFn, + contentEmbeddingAnnCandidateSource + ) + } +} + +@Singleton +class ContentAnnTweetBasedCandidatePipelineConfig[Query <: PipelineQuery] @Inject() ( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long], + contentEmbeddingAnnCandidateSource: ContentEmbeddingAnnCandidateSource) + extends CandidatePipelineConfig[ + Query, + ContentEmbeddingAnnQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentEmbeddingAnn) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "EnableContentEmbeddingAnnTweets", + param = EnableContentEmbeddingAnnTweets + ) + ) + + private def signalFnSelector(query: PipelineQuery): Seq[Long] = { + signalFn(query) + } + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + ContentEmbeddingAnnQuery + ] = { + ContentEmbeddingAnnQueryTransformer( + TransformerIdentifier("ContentEmbeddingAnnTweetBased"), + query => signalFnSelector(query), + ContentEmbeddingAnnParams.MinScoreThreshold, + ContentEmbeddingAnnParams.MaxScoreThreshold, + ContentEmbeddingAnnParams.NumberOfCandidatesPerPost, + ContentEmbeddingAnnParams.IncludeMediaSource, + ContentEmbeddingAnnParams.IncludeTextSource, + ContentEmbeddingAnnParams.DecayByCountry + ) + } + + override def candidateSource: CandidateSource[ + ContentEmbeddingAnnQuery, + TweetMixerCandidate + ] = contentEmbeddingAnnCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => TweetCandidate(id = candidate.tweetId) } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(threshold = 95, notificationType = BusinessHours)(ForYouGroupMap), + defaultEmptyResponseRateAlert()(ForYouGroupMap) + ) +} + +case class ContentEmbeddingAnnQueryTransformer( + override val identifier: TransformerIdentifier, + signalsFn: PipelineQuery => Seq[Long], + minScoreParam: FSBoundedParam[Double], + maxScoreParam: FSBoundedParam[Double], + numCandidatesPerPost: FSBoundedParam[Int], + includeMediaSourceParam: FSParam[Boolean], + includeTextSourceParam: FSParam[Boolean], + decayByCountryParam: FSParam[Boolean]) + extends CandidatePipelineQueryTransformer[PipelineQuery, ContentEmbeddingAnnQuery] { + override def transform(inputQuery: PipelineQuery): ContentEmbeddingAnnQuery = { + val minScore = inputQuery.params(minScoreParam) + val maxScore = inputQuery.params(maxScoreParam) + val numCandidates = inputQuery.params(numCandidatesPerPost) + val countryCode = inputQuery.clientContext.countryCode.getOrElse("US") + val languageCode = inputQuery.clientContext.languageCode.getOrElse("en") + val decayByCountry = inputQuery.params(decayByCountryParam) + val includeMediaSource = inputQuery.params(includeMediaSourceParam) + val includeTextSource = inputQuery.params(includeTextSourceParam) + ContentEmbeddingAnnQuery( + signalsFn(inputQuery), + numCandidates, + minScore, + maxScore, + countryCode, + languageCode, + decayByCountry, + includeMediaSource, + includeTextSource + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRTweetTweetCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRTweetTweetCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..f7625dda3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRTweetTweetCandidatePipelineConfigFactory.scala @@ -0,0 +1,196 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalTweetTweetEmbeddingANNCandidateSource +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.gate.MinTimeSinceLastRequestGate +import com.twitter.tweet_mixer.functional_component.gate.ProbablisticPassGate +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.ContentMediaEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentMediaEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.DeepRetrievalTweetEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRTweetTweetEnableRandomEmbedding +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRTweetTweetEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRTweetTweetMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRTweetTweetScoreThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRTweetTweetVectorDBCollectionName +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import com.twitter.tweet_mixer.utils.Utils +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ContentExplorationDRTweetTweetCandidatePipelineConfigFactory @Inject() ( + deepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory: DeepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory, + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory: DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + contentExplorationEmbeddingQueryFeatureHydratorFactory: ContentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory: ContentMediaEmbeddingQueryFeatureHydratorFactory, + statsReceiver: StatsReceiver) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): ContentExplorationDRTweetTweetCandidatePipelineConfig[Query] = { + val candidateSource = deepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory.build( + CandidatePipelineConstants.ContentExplorationDRTweetTweet + ) + new ContentExplorationDRTweetTweetCandidatePipelineConfig( + candidateSource, + signalFn, + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + contentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory, + identifierPrefix, + statsReceiver) + } +} + +class ContentExplorationDRTweetTweetCandidatePipelineConfig[Query <: PipelineQuery]( + deepRetrievalTweetTweetEmbeddingANNCandidateSource: DeepRetrievalTweetTweetEmbeddingANNCandidateSource, + signalFn: PipelineQuery => Seq[Long], + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory: DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + contentExplorationEmbeddingQueryFeatureHydratorFactory: ContentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory: ContentMediaEmbeddingQueryFeatureHydratorFactory, + identifierPrefix: String, + statsReceiver: StatsReceiver +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRTweetTweet) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "ContentExplorationDeepRetrievalTweetTweetSimilarity", + param = ContentExplorationDRTweetTweetEnabled + ), + MinTimeSinceLastRequestGate, + ProbablisticPassGate + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ), + contentExplorationEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ), + contentMediaEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ) + ) + + private val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetIdsLenStats = scopedStats.stat("tweetIdsLen") + private val embeddingLenStats = scopedStats.stat("embeddingLen") + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + DRMultipleANNQuery + ] = { query => + val tweetIds = signalFn(query) + val collectionName = + query.params(ContentExplorationDRTweetTweetVectorDBCollectionName) + val maxCandidates = + query.params(ContentExplorationDRTweetTweetMaxCandidates) + val scoreThreshold = + query.params(ContentExplorationDRTweetTweetScoreThreshold) + val useRandomEmbedding = + query.params(ContentExplorationDRTweetTweetEnableRandomEmbedding) + + val embeddings: Map[Long, Seq[Int]] = + query.features + .flatMap(_.get(DeepRetrievalTweetEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Int]]) + val textEmbeddingMap: Map[Long, Seq[Double]] = + query.features + .flatMap(_.get(ContentExplorationEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Double]]) + val mediaEmbeddingMap: Map[Long, Seq[Double]] = + query.features + .flatMap(_.get(ContentMediaEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Double]]) + val filteredEmbeddings = embeddings.filter { + case (tweetId, _) => textEmbeddingMap.contains(tweetId) && mediaEmbeddingMap.contains(tweetId) + } + + val tweetSize: Int = tweetIds.size + val embeddingSize: Int = embeddings.size + tweetIdsLenStats.add(tweetSize) + embeddingLenStats.add(embeddingSize) + + val tweetsANNKeys: Seq[DRANNKey] = filteredEmbeddings match { + case embeddingsMap if embeddingsMap.nonEmpty => + embeddingsMap.map { + case (tweetId, embeddingSeq) => + DRANNKey( + id = tweetId, + embedding = + if (useRandomEmbedding) + Some(Utils.generateRandomIntBits(embeddingSeq)) + else Some(embeddingSeq), + collectionName = collectionName, + maxCandidates = maxCandidates, + scoreThreshold = scoreThreshold, + tier = Some("tier1") + ) + }.toSeq + case _ => + Seq.empty[DRANNKey] + } + + DRMultipleANNQuery( + annKeys = tweetsANNKeys, + enableCache = true + ) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = deepRetrievalTweetTweetEmbeddingANNCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRTweetTweetTierTwoCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRTweetTweetTierTwoCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..f4ad9973d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRTweetTweetTierTwoCandidatePipelineConfigFactory.scala @@ -0,0 +1,195 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalTweetTweetEmbeddingANNCandidateSource +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.gate.MinTimeSinceLastRequestGate +import com.twitter.tweet_mixer.functional_component.gate.ProbablisticPassGate +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.ContentMediaEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentMediaEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.DeepRetrievalTweetEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRTweetTweetEnableRandomEmbedding +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRTweetTweetTierTwoEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRTweetTweetTierTwoMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRTweetTweetTierTwoScoreThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRTweetTweetTierTwoVectorDBCollectionName +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import com.twitter.tweet_mixer.utils.Utils +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ContentExplorationDRTweetTweetTierTwoCandidatePipelineConfigFactory @Inject() ( + deepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory: DeepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory, + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory: DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + contentExplorationEmbeddingQueryFeatureHydratorFactory: ContentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory: ContentMediaEmbeddingQueryFeatureHydratorFactory, + statsReceiver: StatsReceiver) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): ContentExplorationDRTweetTweetTierTwoCandidatePipelineConfig[Query] = { + val candidateSource = deepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory.build( + CandidatePipelineConstants.ContentExplorationDRTweetTweetTierTwo + ) + new ContentExplorationDRTweetTweetTierTwoCandidatePipelineConfig( + candidateSource, + signalFn, + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + contentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory, + identifierPrefix, + statsReceiver) + } +} + +class ContentExplorationDRTweetTweetTierTwoCandidatePipelineConfig[Query <: PipelineQuery]( + deepRetrievalTweetTweetEmbeddingANNCandidateSource: DeepRetrievalTweetTweetEmbeddingANNCandidateSource, + signalFn: PipelineQuery => Seq[Long], + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory: DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + contentExplorationEmbeddingQueryFeatureHydratorFactory: ContentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory: ContentMediaEmbeddingQueryFeatureHydratorFactory, + identifierPrefix: String, + statsReceiver: StatsReceiver +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRTweetTweetTierTwo) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "ContentExplorationDeepRetrievalTweetTweetSimilarityTier2", + param = ContentExplorationDRTweetTweetTierTwoEnabled + ), + MinTimeSinceLastRequestGate, + ProbablisticPassGate + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ), + contentExplorationEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ), + contentMediaEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ) + ) + + private val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetIdsLenStats = scopedStats.stat("tweetIdsLen") + private val embeddingLenStats = scopedStats.stat("embeddingLen") + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + DRMultipleANNQuery + ] = { query => + val tweetIds = signalFn(query) + val collectionName = + query.params(ContentExplorationDRTweetTweetTierTwoVectorDBCollectionName) + val maxCandidates = + query.params(ContentExplorationDRTweetTweetTierTwoMaxCandidates) + val scoreThreshold = + query.params(ContentExplorationDRTweetTweetTierTwoScoreThreshold) + val useRandomEmbedding = + query.params(ContentExplorationDRTweetTweetEnableRandomEmbedding) + + val embeddings: Map[Long, Seq[Int]] = + query.features + .flatMap(_.get(DeepRetrievalTweetEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Int]]) + val textEmbeddingMap: Map[Long, Seq[Double]] = + query.features + .flatMap(_.get(ContentExplorationEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Double]]) + val mediaEmbeddingMap: Map[Long, Seq[Double]] = + query.features + .flatMap(_.get(ContentMediaEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Double]]) + val filteredEmbeddings = embeddings.filter { + case (tweetId, _) => textEmbeddingMap.contains(tweetId) && mediaEmbeddingMap.contains(tweetId) + } + + val tweetSize: Int = tweetIds.size + val embeddingSize: Int = embeddings.size + tweetIdsLenStats.add(tweetSize) + embeddingLenStats.add(embeddingSize) + + val tweetsANNKeys: Seq[DRANNKey] = filteredEmbeddings match { + case embeddingsMap if embeddingsMap.nonEmpty => + embeddingsMap.map { + case (tweetId, embeddingSeq) => + DRANNKey( + id = tweetId, + embedding = + if (useRandomEmbedding) + Some(Utils.generateRandomIntBits(embeddingSeq)) + else Some(embeddingSeq), + collectionName = collectionName, + maxCandidates = maxCandidates, + scoreThreshold = scoreThreshold, + tier = Some("tier2") + ) + }.toSeq + case _ => Seq.empty[DRANNKey] + } + + DRMultipleANNQuery( + annKeys = tweetsANNKeys, + enableCache = true + ) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = deepRetrievalTweetTweetEmbeddingANNCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRUserTweetCandidatePipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRUserTweetCandidatePipelineConfig.scala new file mode 100644 index 000000000..9bfcc7b64 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRUserTweetCandidatePipelineConfig.scala @@ -0,0 +1,95 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalUserTweetANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationDRUserEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationDRUserEmbeddingQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRUserTweetEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRUserTweetVectorDBCollectionName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRUserTweetMaxCandidates +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ContentExplorationDRUserTweetCandidatePipelineConfig @Inject() ( + contentExplorationDRUserEmbeddingFeatureHydrator: ContentExplorationDRUserEmbeddingQueryFeatureHydrator, + deepRetrievalUserTweetANNCandidateSourceFactory: DeepRetrievalUserTweetANNCandidateSourceFactory, + identifierPrefix: String) + extends CandidatePipelineConfig[ + PipelineQuery, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRUserTweet) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "ContentExplorationDeepRetrievalUserTweetSimilarity", + param = ContentExplorationDRUserTweetEnabled + ) + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + contentExplorationDRUserEmbeddingFeatureHydrator, + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + DRMultipleANNQuery + ] = { query => + val defaultKey = DRANNKey( + id = query.getRequiredUserId, + embedding = query.features + .flatMap(_.getOrElse(ContentExplorationDRUserEmbeddingFeature, None)), + collectionName = query.params(ContentExplorationDRUserTweetVectorDBCollectionName), + maxCandidates = query.params(ContentExplorationDRUserTweetMaxCandidates), + tier = Some("tier1") + ) + val keys = Seq(defaultKey) + DRMultipleANNQuery(keys, false) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = deepRetrievalUserTweetANNCandidateSourceFactory.build( + CandidatePipelineConstants.ContentExplorationDRUserTweet) + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert()(ForYouGroupMap), + defaultEmptyResponseRateAlert()(ForYouGroupMap) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRUserTweetTierTwoCandidatePipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRUserTweetTierTwoCandidatePipelineConfig.scala new file mode 100644 index 000000000..33002a1d9 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationDRUserTweetTierTwoCandidatePipelineConfig.scala @@ -0,0 +1,95 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalUserTweetANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationDRUserEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationDRUserEmbeddingQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRUserTweetTierTwoEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRUserTweetVectorDBCollectionName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRUserTweetTierTwoMaxCandidates +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ContentExplorationDRUserTweetTierTwoCandidatePipelineConfig @Inject() ( + contentExplorationDRUserEmbeddingFeatureHydrator: ContentExplorationDRUserEmbeddingQueryFeatureHydrator, + deepRetrievalUserTweetANNCandidateSourceFactory: DeepRetrievalUserTweetANNCandidateSourceFactory, + identifierPrefix: String) + extends CandidatePipelineConfig[ + PipelineQuery, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRUserTweetTierTwo) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "ContentExplorationDeepRetrievalUserTweetSimilarityTierTwo", + param = ContentExplorationDRUserTweetTierTwoEnabled + ) + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + contentExplorationDRUserEmbeddingFeatureHydrator, + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + DRMultipleANNQuery + ] = { query => + val defaultKey = DRANNKey( + id = query.getRequiredUserId, + embedding = query.features + .flatMap(_.getOrElse(ContentExplorationDRUserEmbeddingFeature, None)), + collectionName = query.params(ContentExplorationDRUserTweetVectorDBCollectionName), + maxCandidates = query.params(ContentExplorationDRUserTweetTierTwoMaxCandidates), + tier = Some("tier2") + ) + val keys = Seq(defaultKey) + DRMultipleANNQuery(keys, false) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = deepRetrievalUserTweetANNCandidateSourceFactory.build( + CandidatePipelineConstants.ContentExplorationDRUserTweetTierTwo) + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert()(ForYouGroupMap), + defaultEmptyResponseRateAlert()(ForYouGroupMap) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEmbeddingSimilarityCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEmbeddingSimilarityCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..83f47254d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEmbeddingSimilarityCandidatePipelineConfigFactory.scala @@ -0,0 +1,208 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.EmbeddingANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.EmbeddingANNCandidateSource +import com.twitter.tweet_mixer.candidate_source.ndr_ann.EmbeddingANNCandidateSourceFactory +import com.twitter.tweet_mixer.candidate_source.ndr_ann.EmbeddingMultipleANNQuery +import com.twitter.tweet_mixer.functional_component.gate.MinTimeSinceLastRequestGate +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentMediaEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.MultimodalEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.ContentMediaEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.MultimodalEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationEmbeddingANNMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationEmbeddingANNScoreThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationEmbeddingSimilarityEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationMediaEmbeddingSimilarityEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationMultimodalEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationVectorDBCollectionName +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ContentExplorationEmbeddingSimilarityCandidatePipelineConfigFactory @Inject() ( + contentEmbeddingANNCandidateSourceFactory: EmbeddingANNCandidateSourceFactory, + contentExplorationEmbeddingQueryFeatureHydratorFactory: ContentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory: ContentMediaEmbeddingQueryFeatureHydratorFactory, + multimodalEmbeddingQueryFeatureHydratorFactory: MultimodalEmbeddingQueryFeatureHydratorFactory, + statsReceiver: StatsReceiver) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): ContentExplorationEmbeddingSimilarityCandidatePipelineConfig[Query] = { + val candidateSource = contentEmbeddingANNCandidateSourceFactory.build( + CandidatePipelineConstants.ContentExplorationEmbeddingSimilarity + ) + new ContentExplorationEmbeddingSimilarityCandidatePipelineConfig( + candidateSource, + signalFn, + contentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory, + multimodalEmbeddingQueryFeatureHydratorFactory, + identifierPrefix, + statsReceiver) + } +} + +class ContentExplorationEmbeddingSimilarityCandidatePipelineConfig[Query <: PipelineQuery]( + contentEmbeddingANNCandidateSource: EmbeddingANNCandidateSource, + signalFn: PipelineQuery => Seq[Long], + contentExplorationEmbeddingQueryFeatureHydratorFactory: ContentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory: ContentMediaEmbeddingQueryFeatureHydratorFactory, + multimodalEmbeddingQueryFeatureHydratorFactory: MultimodalEmbeddingQueryFeatureHydratorFactory, + identifierPrefix: String, + statsReceiver: StatsReceiver +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + EmbeddingMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationEmbeddingSimilarity) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "ContentExplorationEmbeddingSimilarity", + param = ContentExplorationEmbeddingSimilarityEnabled + ), + MinTimeSinceLastRequestGate + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + contentExplorationEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ), + contentMediaEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ), + multimodalEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ) + ) + + private val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetIdsLenStats = scopedStats.stat("tweetIdsLen") + private val embeddingLenStats = scopedStats.stat("embeddingLen") + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + EmbeddingMultipleANNQuery + ] = { query => + val tweetIds = signalFn(query) + val collectionName = query.params(ContentExplorationVectorDBCollectionName) + val maxCandidates = query.params(ContentExplorationEmbeddingANNMaxCandidates) + val scoreThreshold = query.params(ContentExplorationEmbeddingANNScoreThreshold) + val textEmbeddings = query.features + .flatMap(_.getOrElse(ContentExplorationEmbeddingFeature, Some(Map.empty[Long, Seq[Double]]))) + val textEmbeddingMap: Map[Long, Seq[Double]] = + query.features + .flatMap(_.get(ContentExplorationEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Double]]) + + val mediaEmbeddingMap: Map[Long, Seq[Double]] = + query.features + .flatMap(_.get(ContentMediaEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Double]]) + + val multimodalEmbeddingMap: Map[Long, Option[Seq[Double]]] = + query.features + .flatMap(_.get(MultimodalEmbeddingFeature)) + .getOrElse(Map.empty[Long, Option[Seq[Double]]]) + + val filteredMultimodalEmbeddingMap: Map[Long, Seq[Double]] = + multimodalEmbeddingMap + .filter { case (_, embeddingOpt) => embeddingOpt.isDefined } + .map { case (tweetId, embeddingOpt) => tweetId -> embeddingOpt.get } + + val multimodalEmbeddingMapOpt: Option[Map[Long, Seq[Double]]] = + if (filteredMultimodalEmbeddingMap.isEmpty) None else Some(filteredMultimodalEmbeddingMap) + + val mergedEmbeddingMap: Map[Long, Seq[Double]] = + textEmbeddingMap ++ mediaEmbeddingMap + + val mergedEmbeddingMapOpt: Option[Map[Long, Seq[Double]]] = + if (mergedEmbeddingMap.isEmpty) None else Some(mergedEmbeddingMap) + + val embeddings = + if (query.params( + ContentExplorationMediaEmbeddingSimilarityEnabled) && mergedEmbeddingMapOpt.isDefined) + mergedEmbeddingMapOpt + else if (query.params(ContentExplorationMultimodalEnabled)) + multimodalEmbeddingMapOpt + else textEmbeddings + + val tweetSize: Int = tweetIds.size + val embeddingSize: Int = embeddings.map(_.size).getOrElse(0) + tweetIdsLenStats.add(tweetSize) + embeddingLenStats.add(embeddingSize) + + val tweetsANNKeys: Seq[EmbeddingANNKey] = embeddings match { + case Some(embeddingsMap) => + embeddingsMap.map { + case (tweetId, embeddingSeq) => + EmbeddingANNKey( + tweetId, + Some(embeddingSeq), + collectionName, + maxCandidates, + scoreThreshold, + tier = Some("tier1") + ) + }.toSeq + case None => + Seq.empty[EmbeddingANNKey] + } + + EmbeddingMultipleANNQuery( + annKeys = tweetsANNKeys, + enableCache = true + ) + } + + override def candidateSource: CandidateSource[ + EmbeddingMultipleANNQuery, + TweetMixerCandidate + ] = contentEmbeddingANNCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..5a611fcd9 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfigFactory.scala @@ -0,0 +1,210 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.EmbeddingANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.EmbeddingANNCandidateSource +import com.twitter.tweet_mixer.candidate_source.ndr_ann.EmbeddingANNCandidateSourceFactory +import com.twitter.tweet_mixer.candidate_source.ndr_ann.EmbeddingMultipleANNQuery +import com.twitter.tweet_mixer.functional_component.gate.MinTimeSinceLastRequestGate +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentMediaEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.MultimodalEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.ContentMediaEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.MultimodalEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationEmbeddingSimilarityTierTwoEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationEmbeddingSimilarityTierTwoMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationEmbeddingSimilarityTierTwoScoreThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationEmbeddingSimilarityTierTwoVectorDBCollectionName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationMediaEmbeddingSimilarityEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationMultimodalEnabled +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ContentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfigFactory @Inject() ( + contentEmbeddingANNCandidateSourceFactory: EmbeddingANNCandidateSourceFactory, + contentExplorationEmbeddingQueryFeatureHydratorFactory: ContentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory: ContentMediaEmbeddingQueryFeatureHydratorFactory, + multimodalEmbeddingQueryFeatureHydratorFactory: MultimodalEmbeddingQueryFeatureHydratorFactory, + statsReceiver: StatsReceiver) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): ContentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfig[Query] = { + val candidateSource = contentEmbeddingANNCandidateSourceFactory.build( + CandidatePipelineConstants.ContentExplorationEmbeddingSimilarityTierTwo + ) + new ContentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfig( + candidateSource, + signalFn, + contentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory, + multimodalEmbeddingQueryFeatureHydratorFactory, + identifierPrefix, + statsReceiver) + } +} + +class ContentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfig[Query <: PipelineQuery]( + contentEmbeddingANNCandidateSource: EmbeddingANNCandidateSource, + signalFn: PipelineQuery => Seq[Long], + contentExplorationEmbeddingQueryFeatureHydratorFactory: ContentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory: ContentMediaEmbeddingQueryFeatureHydratorFactory, + multimodalEmbeddingQueryFeatureHydratorFactory: MultimodalEmbeddingQueryFeatureHydratorFactory, + identifierPrefix: String, + statsReceiver: StatsReceiver +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + EmbeddingMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationEmbeddingSimilarityTierTwo) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "ContentExplorationEmbeddingSimilarityTier2", + param = ContentExplorationEmbeddingSimilarityTierTwoEnabled + ), + MinTimeSinceLastRequestGate + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + contentExplorationEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ), + contentMediaEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ), + multimodalEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ) + ) + + private val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetIdsLenStats = scopedStats.stat("tweetIdsLen") + private val embeddingLenStats = scopedStats.stat("embeddingLen") + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + EmbeddingMultipleANNQuery + ] = { query => + val tweetIds = signalFn(query) + val collectionName = + query.params(ContentExplorationEmbeddingSimilarityTierTwoVectorDBCollectionName) + val maxCandidates = query.params(ContentExplorationEmbeddingSimilarityTierTwoMaxCandidates) + val scoreThreshold = query.params(ContentExplorationEmbeddingSimilarityTierTwoScoreThreshold) + val textEmbeddings = query.features + .flatMap(_.getOrElse(ContentExplorationEmbeddingFeature, Some(Map.empty[Long, Seq[Double]]))) + val textEmbeddingMap: Map[Long, Seq[Double]] = + query.features + .flatMap(_.get(ContentExplorationEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Double]]) + + val mediaEmbeddingMap: Map[Long, Seq[Double]] = + query.features + .flatMap(_.get(ContentMediaEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Double]]) + + val multimodalEmbeddingMap: Map[Long, Option[Seq[Double]]] = + query.features + .flatMap(_.get(MultimodalEmbeddingFeature)) + .getOrElse(Map.empty[Long, Option[Seq[Double]]]) + + val filteredMultimodalEmbeddingMap: Map[Long, Seq[Double]] = + multimodalEmbeddingMap + .filter { case (_, embeddingOpt) => embeddingOpt.isDefined } + .map { case (tweetId, embeddingOpt) => tweetId -> embeddingOpt.get } + + val multimodalEmbeddingMapOpt: Option[Map[Long, Seq[Double]]] = + if (filteredMultimodalEmbeddingMap.isEmpty) None else Some(filteredMultimodalEmbeddingMap) + + val mergedEmbeddingMap: Map[Long, Seq[Double]] = + textEmbeddingMap ++ mediaEmbeddingMap + + val mergedEmbeddingMapOpt: Option[Map[Long, Seq[Double]]] = + if (mergedEmbeddingMap.isEmpty) None else Some(mergedEmbeddingMap) + + val embeddings = + if (query.params( + ContentExplorationMediaEmbeddingSimilarityEnabled) && mergedEmbeddingMapOpt.isDefined) + mergedEmbeddingMapOpt + else if (query.params( + ContentExplorationMultimodalEnabled) && multimodalEmbeddingMapOpt.isDefined) + multimodalEmbeddingMapOpt + else textEmbeddings + + val tweetSize: Int = tweetIds.size + val embeddingSize: Int = embeddings.map(_.size).getOrElse(0) + tweetIdsLenStats.add(tweetSize) + embeddingLenStats.add(embeddingSize) + + val tweetsANNKeys: Seq[EmbeddingANNKey] = embeddings match { + case Some(embeddingsMap) => + embeddingsMap.map { + case (tweetId, embeddingSeq) => + EmbeddingANNKey( + tweetId, + Some(embeddingSeq), + collectionName, + maxCandidates, + scoreThreshold, + tier = Some("tier2") + ) + }.toSeq + case None => + Seq.empty[EmbeddingANNKey] + } + + EmbeddingMultipleANNQuery( + annKeys = tweetsANNKeys, + enableCache = true + ) + } + + override def candidateSource: CandidateSource[ + EmbeddingMultipleANNQuery, + TweetMixerCandidate + ] = contentEmbeddingANNCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEvergreenDRTweetTweetCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEvergreenDRTweetTweetCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..f8be66c06 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationEvergreenDRTweetTweetCandidatePipelineConfigFactory.scala @@ -0,0 +1,185 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalTweetTweetEmbeddingANNCandidateSource +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.gate.MinTimeSinceLastRequestGate +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentExplorationEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.ContentMediaEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.ContentMediaEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.DeepRetrievalTweetEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationEvergreenDRTweetTweetEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationEvergreenDRTweetTweetMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationEvergreenDRTweetTweetScoreThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationEvergreenDRTweetTweetVectorDBCollectionName +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ContentExplorationEvergreenDRTweetTweetCandidatePipelineConfigFactory @Inject() ( + deepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory: DeepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory, + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory: DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + contentExplorationEmbeddingQueryFeatureHydratorFactory: ContentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory: ContentMediaEmbeddingQueryFeatureHydratorFactory, + statsReceiver: StatsReceiver) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): ContentExplorationEvergreenDRTweetTweetCandidatePipelineConfig[Query] = { + val candidateSource = deepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory.build( + CandidatePipelineConstants.ContentExplorationEvergreenDRTweetTweet + ) + new ContentExplorationEvergreenDRTweetTweetCandidatePipelineConfig( + candidateSource, + signalFn, + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + contentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory, + identifierPrefix, + statsReceiver) + } +} + +class ContentExplorationEvergreenDRTweetTweetCandidatePipelineConfig[Query <: PipelineQuery]( + deepRetrievalTweetTweetEmbeddingANNCandidateSource: DeepRetrievalTweetTweetEmbeddingANNCandidateSource, + signalFn: PipelineQuery => Seq[Long], + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory: DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + contentExplorationEmbeddingQueryFeatureHydratorFactory: ContentExplorationEmbeddingQueryFeatureHydratorFactory, + contentMediaEmbeddingQueryFeatureHydratorFactory: ContentMediaEmbeddingQueryFeatureHydratorFactory, + identifierPrefix: String, + statsReceiver: StatsReceiver +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationEvergreenDRTweetTweet) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "ContentExplorationEvergreenDeepRetrievalTweetTweetSimilarity", + param = ContentExplorationEvergreenDRTweetTweetEnabled + ), + MinTimeSinceLastRequestGate + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ), + contentExplorationEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ), + contentMediaEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ) + ) + + private val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetIdsLenStats = scopedStats.stat("tweetIdsLen") + private val embeddingLenStats = scopedStats.stat("embeddingLen") + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + DRMultipleANNQuery + ] = { query => + val tweetIds = signalFn(query) + val collectionName = + query.params(ContentExplorationEvergreenDRTweetTweetVectorDBCollectionName) + val maxCandidates = + query.params(ContentExplorationEvergreenDRTweetTweetMaxCandidates) + val scoreThreshold = + query.params(ContentExplorationEvergreenDRTweetTweetScoreThreshold) + val embeddings: Map[Long, Seq[Int]] = + query.features + .flatMap(_.get(DeepRetrievalTweetEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Int]]) + val textEmbeddingMap: Map[Long, Seq[Double]] = + query.features + .flatMap(_.get(ContentExplorationEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Double]]) + val mediaEmbeddingMap: Map[Long, Seq[Double]] = + query.features + .flatMap(_.get(ContentMediaEmbeddingFeature)) + .getOrElse(Map.empty[Long, Seq[Double]]) + val filteredEmbeddings = embeddings.filter { + case (tweetId, _) => textEmbeddingMap.contains(tweetId) && mediaEmbeddingMap.contains(tweetId) + } + + val tweetSize: Int = tweetIds.size + val embeddingSize: Int = embeddings.size + tweetIdsLenStats.add(tweetSize) + embeddingLenStats.add(embeddingSize) + + val tweetsANNKeys: Seq[DRANNKey] = filteredEmbeddings match { + case embeddingsMap if embeddingsMap.nonEmpty => + embeddingsMap.map { + case (tweetId, embeddingSeq) => + DRANNKey( + id = tweetId, + embedding = Some(embeddingSeq), + collectionName = collectionName, + maxCandidates = maxCandidates, + scoreThreshold = scoreThreshold + ) + }.toSeq + case _ => + Seq.empty[DRANNKey] + } + + DRMultipleANNQuery( + annKeys = tweetsANNKeys, + enableCache = true + ) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = deepRetrievalTweetTweetEmbeddingANNCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationSimclusterColdCandidatePipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationSimclusterColdCandidatePipelineConfig.scala new file mode 100644 index 000000000..e712117a3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ContentExplorationSimclusterColdCandidatePipelineConfig.scala @@ -0,0 +1,74 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SimclusterColdPostsCandidateSource +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SimclusterColdPostsQuery +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.CoreProductGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.BusinessHours +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationSimclusterColdPostsEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationSimclusterColdPostsMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationSimclusterColdPostsPostsPerSimcluster +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ContentExplorationSimclusterColdCandidatePipelineConfig @Inject() ( + simclusterColdPostsCandidateSource: SimclusterColdPostsCandidateSource, + identifierPrefix: String) + extends CandidatePipelineConfig[ + PipelineQuery, + SimclusterColdPostsQuery, + Long, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationSimclusterColdPosts) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "ContentExplorationSimclusterColdPosts", + param = ContentExplorationSimclusterColdPostsEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + SimclusterColdPostsQuery + ] = { query => + SimclusterColdPostsQuery( + userId = query.getRequiredUserId, + postsPerSimcluster = query.params(ContentExplorationSimclusterColdPostsPostsPerSimcluster), + maxCandidates = query.params(ContentExplorationSimclusterColdPostsMaxCandidates) + ) + } + + override def candidateSource: CandidateSource[ + SimclusterColdPostsQuery, + Long + ] = simclusterColdPostsCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + Long, + TweetCandidate + ] = { tweetId => + TweetCandidate(id = tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(threshold = 95, notificationType = BusinessHours)(CoreProductGroupMap), + defaultEmptyResponseRateAlert()(CoreProductGroupMap) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ControlAiTopicCandidatePipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ControlAiTopicCandidatePipelineConfig.scala new file mode 100644 index 000000000..580c25221 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/ControlAiTopicCandidatePipelineConfig.scala @@ -0,0 +1,94 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_pipeline.ControlAiTopicCandidatePipelineConfig.Identifier +import com.twitter.tweet_mixer.candidate_source.text_embedding_ann.TextEmbeddingCandidateSource +import com.twitter.tweet_mixer.candidate_source.text_embedding_ann.TextEmbeddingQuery +import com.twitter.tweet_mixer.functional_component.hydrator.ControlAiTopicEmbeddingQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.hydrator.ControlAiTopicEmbeddings +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ControlAiEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ControlAiTopicEmbeddingANNMaxCandidates +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ControlAiTopicCandidatePipelineConfig @Inject() ( + controlAiTopicEmbeddingQueryFeatureHydrator: ControlAiTopicEmbeddingQueryFeatureHydrator, + textEmbeddingCandidateSource: TextEmbeddingCandidateSource) + extends CandidatePipelineConfig[ + PipelineQuery, + Seq[TextEmbeddingQuery], + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = Identifier + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "ControlAiTopic", + param = ControlAiEnabled + ) + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = + Seq(controlAiTopicEmbeddingQueryFeatureHydrator) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + Seq[TextEmbeddingQuery] + ] = { query => + val totalMaxCandidates = query.params(ControlAiTopicEmbeddingANNMaxCandidates) + + query.features + .map(q => q.getOrElse(ControlAiTopicEmbeddings, Seq.empty)) + .map { embeddings => + val maxCandidates = + if (embeddings.nonEmpty) Math.max(totalMaxCandidates / embeddings.size, 50) else 0 + embeddings.map { embedding => + TextEmbeddingQuery(vector = embedding, limit = maxCandidates) + } + }.getOrElse(Seq.empty) + } + + override def candidateSource: CandidateSource[ + Seq[TextEmbeddingQuery], + TweetMixerCandidate + ] = textEmbeddingCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert()(ForYouGroupMap), + ) +} + +object ControlAiTopicCandidatePipelineConfig { + val Identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + CandidatePipelineConstants.ControlAiTopicTweets) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/CuratedUserTlsPerLangaugeCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/CuratedUserTlsPerLangaugeCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..a9c5cf84d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/CuratedUserTlsPerLangaugeCandidatePipelineConfigFactory.scala @@ -0,0 +1,75 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelineservice.{thriftscala => tls} +import com.twitter.tweet_mixer.candidate_source.curated_user_tls_per_language.CuratedUserTlsPerLanguageCandidateSource +import com.twitter.tweet_mixer.param.CuratedUserTlsPerLanguageParams._ +import com.twitter.tweet_mixer.product.home_recommended_tweets.model.request.HomeRecommendedTweetsQuery +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants + +import javax.inject.Inject +import javax.inject.Singleton +@Singleton +class CuratedUserTlsPerLanguageCandidatePipelineConfigFactory @Inject() ( + curatedUserTlsPerLanguageCandidateSource: CuratedUserTlsPerLanguageCandidateSource) { + def build[Query <: HomeRecommendedTweetsQuery]( + identifierPrefix: String + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): CuratedUserTlsPerLanguageCandidateSourceConfig[Query] = { + new CuratedUserTlsPerLanguageCandidateSourceConfig( + identifierPrefix, + curatedUserTlsPerLanguageCandidateSource) + } +} + +class CuratedUserTlsPerLanguageCandidateSourceConfig[Query <: HomeRecommendedTweetsQuery]( + identifierPrefix: String, + curatedUserTlsPerLanguageCandidateSource: CuratedUserTlsPerLanguageCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + Seq[tls.TimelineQuery], + TweetCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.CuratedUserTlsPerLanguage) + + override val supportedClientParam: Option[FSParam[Boolean]] = Some( + CuratedUserTlsPerLanguageTweetsEnable) + + private val MaxTweetsToFetchPerAuthor = 10 + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + Seq[tls.TimelineQuery] + ] = { query => + val authorIds = query.params(CuratedUserTlsPerLanguageTweetsAuthorListParam) + + authorIds.map { authorId => + tls.TimelineQuery( + timelineType = tls.TimelineType.User, + timelineId = authorId, + maxCount = MaxTweetsToFetchPerAuthor.toShort, + options = Some(tls.TimelineQueryOptions(query.clientContext.userId)), + ) + } + } + + override def candidateSource: CandidateSource[Seq[tls.TimelineQuery], TweetCandidate] = + curatedUserTlsPerLanguageCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetCandidate, + TweetCandidate + ] = identity +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..fcc8ae959 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfigFactory.scala @@ -0,0 +1,153 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalTweetTweetEmbeddingANNCandidateSource +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.gate.MinTimeSinceLastRequestGate +import com.twitter.tweet_mixer.functional_component.hydrator.DeepRetrievalTweetEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetEmbeddingANNEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetEmbeddingANNMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalI2iEmbVectorDBCollectionName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetEmbeddingANNScoreThreshold +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class DeepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfigFactory @Inject() ( + deepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory: DeepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory, + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory: DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + statsReceiver: StatsReceiver) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): DeepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfig[Query] = { + val candidateSource = deepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory.build( + CandidatePipelineConstants.DeepRetrievalTweetTweetEmbeddingSimilarity + ) + new DeepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfig( + candidateSource, + signalFn, + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + identifierPrefix, + statsReceiver) + } +} + +class DeepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfig[Query <: PipelineQuery]( + deepRetrievalTweetTweetEmbeddingANNCandidateSource: DeepRetrievalTweetTweetEmbeddingANNCandidateSource, + signalFn: PipelineQuery => Seq[Long], + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory: DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + identifierPrefix: String, + statsReceiver: StatsReceiver +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.DeepRetrievalTweetTweetEmbeddingSimilarity) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "DeepRetrievalTweetTweetEmbeddingANN", + param = DeepRetrievalTweetTweetEmbeddingANNEnabled + ), + MinTimeSinceLastRequestGate + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + deepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.build( + signalFn + ) + ) + + private val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetIdsLenStats = scopedStats.stat("tweetIdsLen") + private val embeddingLenStats = scopedStats.stat("embeddingLen") + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + DRMultipleANNQuery + ] = { query => + val tweetIds = signalFn(query) + val collectionName = query.params(DeepRetrievalI2iEmbVectorDBCollectionName) + val maxCandidates = query.params(DeepRetrievalTweetTweetEmbeddingANNMaxCandidates) + val scoreThreshold = query.params(DeepRetrievalTweetTweetEmbeddingANNScoreThreshold) + val embeddings = query.features + .flatMap(_.getOrElse(DeepRetrievalTweetEmbeddingFeature, Some(Map.empty[Long, Seq[Int]]))) + + val tweetSize: Int = tweetIds.size + val embeddingSize: Int = embeddings.map(_.size).getOrElse(0) + tweetIdsLenStats.add(tweetSize) + embeddingLenStats.add(embeddingSize) + + val tweetsANNKeys: Seq[DRANNKey] = embeddings match { + case Some(embeddingsMap) => + embeddingsMap.map { + case (tweetId, embeddingSeq) => + DRANNKey( + tweetId, + Some(embeddingSeq), + collectionName, + maxCandidates, + scoreThreshold + ) + }.toSeq + case None => + Seq.empty[DRANNKey] + } + + DRMultipleANNQuery( + annKeys = tweetsANNKeys, + enableCache = true + ) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = deepRetrievalTweetTweetEmbeddingANNCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..a76cbe9bc --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory.scala @@ -0,0 +1,107 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalTweetTweetANNCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalI2iVectorDBCollectionName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetANNEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetANNMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetANNScoreThreshold +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class DeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory @Inject() ( + deepRetrievalTweetTweetANNCandidateSource: DeepRetrievalTweetTweetANNCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): DeepRetrievalTweetTweetSimilarityCandidatePipelineConfig[Query] = { + new DeepRetrievalTweetTweetSimilarityCandidatePipelineConfig( + deepRetrievalTweetTweetANNCandidateSource, + signalFn, + identifierPrefix) + } +} + +class DeepRetrievalTweetTweetSimilarityCandidatePipelineConfig[Query <: PipelineQuery]( + deepRetrievalTweetTweetANNCandidateSource: DeepRetrievalTweetTweetANNCandidateSource, + signalFn: PipelineQuery => Seq[Long], + identifierPrefix: String +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.DeepRetrievalTweetTweetSimilarity) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "DeepRetrievalTweetTweetANN", + param = DeepRetrievalTweetTweetANNEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + DRMultipleANNQuery + ] = { query => + val tweetIds = signalFn(query) + val collectionName = query.params(DeepRetrievalI2iVectorDBCollectionName) + val maxCandidates = query.params(DeepRetrievalTweetTweetANNMaxCandidates) + val scoreThreshold = query.params(DeepRetrievalTweetTweetANNScoreThreshold) + + DRMultipleANNQuery( + annKeys = tweetIds.map { tweetId => + DRANNKey(tweetId, None, collectionName, maxCandidates, scoreThreshold) + }, + enableCache = true + ) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = deepRetrievalTweetTweetANNCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalUserTweetSimilarityCandidatePipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalUserTweetSimilarityCandidatePipelineConfig.scala new file mode 100644 index 000000000..e06c97a47 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/DeepRetrievalUserTweetSimilarityCandidatePipelineConfig.scala @@ -0,0 +1,131 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalUserTweetANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.hydrator.DeepRetrievalUserEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.DeepRetrievalUserEmbeddingQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.hydrator.GrokCategoriesFeature +import com.twitter.tweet_mixer.functional_component.hydrator.GrokCategoriesQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalCategoricalUserTweetANNEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalEnableGPU +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalIsHighQualityEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalIsLowNegEngRatioEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalUserTweetANNEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalUserTweetANNMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalUserTweetANNScoreThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalVectorDBCollectionName +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class DeepRetrievalUserTweetSimilarityCandidatePipelineConfig @Inject() ( + deepRetrievalUserEmbeddingFeatureHydrator: DeepRetrievalUserEmbeddingQueryFeatureHydrator, + deepRetrievalUserTweetANNCandidateSourceFactory: DeepRetrievalUserTweetANNCandidateSourceFactory, + grokCategoriesQueryFeatureHydrator: GrokCategoriesQueryFeatureHydrator, + identifierPrefix: String) + extends CandidatePipelineConfig[ + PipelineQuery, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.DeepRetrievalUserTweetSimilarity) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "DeepRetrievalUserTweetANN", + param = DeepRetrievalUserTweetANNEnabled + ) + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + deepRetrievalUserEmbeddingFeatureHydrator, + ParamGatedQueryFeatureHydrator( + DeepRetrievalCategoricalUserTweetANNEnabled, + grokCategoriesQueryFeatureHydrator + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + DRMultipleANNQuery + ] = { query => + val defaultKey = DRANNKey( + id = query.getRequiredUserId, + embedding = query.features + .flatMap(_.getOrElse(DeepRetrievalUserEmbeddingFeature, None)), + collectionName = query.params(DeepRetrievalVectorDBCollectionName), + maxCandidates = query.params(DeepRetrievalUserTweetANNMaxCandidates), + scoreThreshold = query.params(DeepRetrievalUserTweetANNScoreThreshold), + enableGPU = query.params(DeepRetrievalEnableGPU), + isHighQuality = if (query.params(DeepRetrievalIsHighQualityEnabled)) Some(true) else None, + isLowNegEngRatio = + if (query.params(DeepRetrievalIsLowNegEngRatioEnabled)) Some(false) else None + ) + val categories = + if (query.params(DeepRetrievalCategoricalUserTweetANNEnabled)) + query.features.get.getOrElse(GrokCategoriesFeature, None) + else None + val keysOptSeq = categories.map(_.map(_.toString)).map { categories => + categories.map { category => + DRANNKey( + id = query.getRequiredUserId, + embedding = query.features + .flatMap(_.getOrElse(DeepRetrievalUserEmbeddingFeature, None)), + collectionName = query.params(DeepRetrievalVectorDBCollectionName), + maxCandidates = query.params(DeepRetrievalUserTweetANNMaxCandidates), + category = Some(category), + isHighQuality = if (query.params(DeepRetrievalIsHighQualityEnabled)) Some(true) else None, + isLowNegEngRatio = + if (query.params(DeepRetrievalIsLowNegEngRatioEnabled)) Some(false) else None + ) + } + } + val keys = keysOptSeq.getOrElse(Seq(defaultKey)) + DRMultipleANNQuery(keys, false) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = deepRetrievalUserTweetANNCandidateSourceFactory.build( + identifierPrefix + CandidatePipelineConstants.DeepRetrievalUserTweetSimilarity) + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert()(ForYouGroupMap), + defaultEmptyResponseRateAlert()(ForYouGroupMap) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EarlybirdInNetworkCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EarlybirdInNetworkCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..d2c137113 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EarlybirdInNetworkCandidatePipelineConfigFactory.scala @@ -0,0 +1,86 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.finagle.thrift.ClientId +import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.HasExcludedIds +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.earlybird_realtime_cg.EarlybirdRealtimeCGTweetCandidateSource +import com.twitter.tweet_mixer.candidate_source.earlybird_realtime_cg.InNetworkRequest +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.tweet_mixer.functional_component.hydrator.HaploliteFeature +import com.twitter.tweet_mixer.functional_component.transformer.EarlybirdInNetworkQueryTransformer +import com.twitter.tweet_mixer.functional_component.transformer.EarlybirdInNetworkResponseFeatureTransformer +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.HaploliteTweetsBasedEnabled +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class EarlybirdInNetworkCandidatePipelineConfigFactory @Inject() ( + earlybirdTweetCandidateSource: EarlybirdRealtimeCGTweetCandidateSource, + clientId: ClientId) { + + def build[Query <: PipelineQuery with HasExcludedIds]( + identifierPrefix: String + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): EarlybirdInNetworkCandidatePipelineConfig[Query] = { + new EarlybirdInNetworkCandidatePipelineConfig( + identifierPrefix, + earlybirdTweetCandidateSource, + clientId) + } +} + +class EarlybirdInNetworkCandidatePipelineConfig[Query <: PipelineQuery with HasExcludedIds]( + identifierPrefix: String, + earlybirdTweetCandidateSource: EarlybirdRealtimeCGTweetCandidateSource, + clientId: ClientId +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + InNetworkRequest, + eb.ThriftSearchResult, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.EarlybirdInNetwork) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "HaploliteTweetsBasedEBEnabled", + param = HaploliteTweetsBasedEnabled + ), + EmptySeqFeatureGate(HaploliteFeature) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + InNetworkRequest + ] = EarlybirdInNetworkQueryTransformer(identifier, Some(clientId.name + ".prod")) + + override def candidateSource: BaseCandidateSource[InNetworkRequest, eb.ThriftSearchResult] = + earlybirdTweetCandidateSource + override val featuresFromCandidateSourceTransformers = Seq( + EarlybirdInNetworkResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + eb.ThriftSearchResult, + TweetCandidate + ] = { candidate => + TweetCandidate( + id = candidate.id + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EventsCandidatePipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EventsCandidatePipelineConfig.scala new file mode 100644 index 000000000..7a3e7b5ad --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EventsCandidatePipelineConfig.scala @@ -0,0 +1,68 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.strato.generated.client.recommendations.simclusters_v2.TopPostsPerEventClientColumn +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.candidate_source.events.EventsCandidateSource +import com.twitter.tweet_mixer.candidate_source.events.EventsRequest +import com.twitter.tweet_mixer.functional_component.gate.AllowLowSignalUserGate +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.CandidateSourceParams.EventsEnabled +import com.twitter.tweet_mixer.param.CandidateSourceParams.EventsIrrelevanceDownrank +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.BusinessHours +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class EventsCandidatePipelineConfig @Inject() ( + eventsCandidateSource: EventsCandidateSource) + extends CandidatePipelineConfig[ + PipelineQuery, + EventsRequest, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier(CandidatePipelineConstants.Events) + + override val supportedClientParam: Option[FSParam[Boolean]] = Some(EventsEnabled) + + override val gates: Seq[Gate[PipelineQuery]] = Seq(AllowLowSignalUserGate) + + override val queryTransformer: CandidatePipelineQueryTransformer[PipelineQuery, EventsRequest] = { + query => + EventsRequest(TopPostsPerEventClientColumn.Nba, query.params(EventsIrrelevanceDownrank)) + } + + override def candidateSource: CandidateSource[EventsRequest, TweetMixerCandidate] = + eventsCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => TweetCandidate(id = candidate.tweetId) } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(threshold = 95, notificationType = BusinessHours)(ForYouGroupMap), + defaultEmptyResponseRateAlert()(ForYouGroupMap) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenDRCrossBorderUserTweetCandidatePipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenDRCrossBorderUserTweetCandidatePipelineConfig.scala new file mode 100644 index 000000000..fa2be467a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenDRCrossBorderUserTweetCandidatePipelineConfig.scala @@ -0,0 +1,94 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalUserTweetANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.hydrator.EvergreenDRUserEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.EvergreenDRUserEmbeddingQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenDRCrossBorderUserTweetEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenDRCrossBorderUserTweetVectorDBCollectionName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenDRCrossBorderUserTweetMaxCandidates +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class EvergreenDRCrossBorderUserTweetCandidatePipelineConfig @Inject() ( + evergreenDRUserEmbeddingQueryFeatureHydrator: EvergreenDRUserEmbeddingQueryFeatureHydrator, + deepRetrievalUserTweetANNCandidateSourceFactory: DeepRetrievalUserTweetANNCandidateSourceFactory, + identifierPrefix: String) + extends CandidatePipelineConfig[ + PipelineQuery, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.EvergreenDRCrossBorderUserTweet) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "EvergreenDeepRetrievalCrossBorderUserTweetSimilarity", + param = EvergreenDRCrossBorderUserTweetEnabled + ) + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + evergreenDRUserEmbeddingQueryFeatureHydrator, + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + DRMultipleANNQuery + ] = { query => + val defaultKey = DRANNKey( + id = query.getRequiredUserId, + embedding = query.features + .flatMap(_.getOrElse(EvergreenDRUserEmbeddingFeature, None)), + collectionName = query.params(EvergreenDRCrossBorderUserTweetVectorDBCollectionName), + maxCandidates = query.params(EvergreenDRCrossBorderUserTweetMaxCandidates) + ) + val keys = Seq(defaultKey) + DRMultipleANNQuery(keys, false) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = deepRetrievalUserTweetANNCandidateSourceFactory.build( + CandidatePipelineConstants.EvergreenDRCrossBorderUserTweet) + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert()(ForYouGroupMap), + defaultEmptyResponseRateAlert()(ForYouGroupMap) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenDRUserTweetCandidatePipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenDRUserTweetCandidatePipelineConfig.scala new file mode 100644 index 000000000..8718bbb97 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenDRUserTweetCandidatePipelineConfig.scala @@ -0,0 +1,94 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DeepRetrievalUserTweetANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.hydrator.EvergreenDRUserEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.EvergreenDRUserEmbeddingQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenDRUserTweetEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenDRUserTweetVectorDBCollectionName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenDRUserTweetMaxCandidates +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class EvergreenDRUserTweetCandidatePipelineConfig @Inject() ( + evergreenDRUserEmbeddingQueryFeatureHydrator: EvergreenDRUserEmbeddingQueryFeatureHydrator, + deepRetrievalUserTweetANNCandidateSourceFactory: DeepRetrievalUserTweetANNCandidateSourceFactory, + identifierPrefix: String) + extends CandidatePipelineConfig[ + PipelineQuery, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.EvergreenDRUserTweet) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "EvergreenDeepRetrievalUserTweetSimilarity", + param = EvergreenDRUserTweetEnabled + ) + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + evergreenDRUserEmbeddingQueryFeatureHydrator, + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + DRMultipleANNQuery + ] = { query => + val defaultKey = DRANNKey( + id = query.getRequiredUserId, + embedding = query.features + .flatMap(_.getOrElse(EvergreenDRUserEmbeddingFeature, None)), + collectionName = query.params(EvergreenDRUserTweetVectorDBCollectionName), + maxCandidates = query.params(EvergreenDRUserTweetMaxCandidates) + ) + val keys = Seq(defaultKey) + DRMultipleANNQuery(keys, false) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = deepRetrievalUserTweetANNCandidateSourceFactory.build( + CandidatePipelineConstants.EvergreenDRUserTweet) + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert()(ForYouGroupMap), + defaultEmptyResponseRateAlert()(ForYouGroupMap) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenVideosCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenVideosCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..5b0b6e48a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/EvergreenVideosCandidatePipelineConfigFactory.scala @@ -0,0 +1,88 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.HistoricalEvergreenVideosCandidateSource +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.EvergreenVideosSearchByUserIdsQuery +import com.twitter.tweet_mixer.functional_component.transformer.EvergreenVideosQueryTransformer +import com.twitter.tweet_mixer.functional_component.transformer.EvergreenVideosResponseFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenVideosEnabled +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class EvergreenVideosCandidatePipelineConfigFactory @Inject() ( + historicalEvergreenVideosCandidateSource: HistoricalEvergreenVideosCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): EvergreenVideosCandidatePipelineConfig[Query] = { + new EvergreenVideosCandidatePipelineConfig( + identifierPrefix, + historicalEvergreenVideosCandidateSource + ) + } +} + +class EvergreenVideosCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + historicalEvergreenVideosCandidateSource: HistoricalEvergreenVideosCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + EvergreenVideosSearchByUserIdsQuery, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.EvergreenVideos) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "EvergreenVideosEnabled", + param = EvergreenVideosEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + EvergreenVideosSearchByUserIdsQuery + ] = EvergreenVideosQueryTransformer(identifier) + + override def candidateSource: CandidateSource[ + EvergreenVideosSearchByUserIdsQuery, + TweetMixerCandidate, + ] = historicalEvergreenVideosCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { tweet => + TweetCandidate( + id = tweet.tweetId + ) + } + + override val featuresFromCandidateSourceTransformers = Seq( + EvergreenVideosResponseFeatureTransformer) + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/HaploliteCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/HaploliteCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..1b756bf1b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/HaploliteCandidatePipelineConfigFactory.scala @@ -0,0 +1,90 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.HasExcludedIds +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.clients.haplolite.TweetTimelineHaploCodec +import com.twitter.timelineservice.model.Tweet +import com.twitter.timelineservice.model.core.TimelineKind +import com.twitter.tweet_mixer.functional_component.hydrator.HaploliteFeature +import com.twitter.tweet_mixer.functional_component.transformer.HaploliteResponseFeatureTransformer +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.HaploliteTweetsBasedEnabled +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HaploliteCandidatePipelineConfigFactory @Inject() (statsReceiver: StatsReceiver) { + private val tweetTimelineHaploCodec = new TweetTimelineHaploCodec(statsReceiver) + def build[Query <: PipelineQuery with HasExcludedIds]( + identifierPrefix: String + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): HaploliteCandidatePipelineConfig[Query] = { + new HaploliteCandidatePipelineConfig(identifierPrefix, tweetTimelineHaploCodec) + } +} + +class HaploliteCandidatePipelineConfig[Query <: PipelineQuery with HasExcludedIds]( + identifierPrefix: String, + codec: TweetTimelineHaploCodec +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + Query, + Tweet, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.Haplolite) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "HaploliteTweets", + param = HaploliteTweetsBasedEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + Query + ] = identity + + private def getTweets(query: PipelineQuery): Seq[Tweet] = { + val haploliteFeature = query.features + .getOrElse(FeatureMap.empty) + .getOrElse(HaploliteFeature, Seq.empty) + val result = codec + .decode(haploliteFeature, TimelineKind.home) + result + } + + override def candidateSource: BaseCandidateSource[Query, Tweet] = + PassthroughCandidateSource( + CandidateSourceIdentifier(identifier.name), + getTweets + ) + + override val featuresFromCandidateSourceTransformers = Seq(HaploliteResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + Tweet, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/MediaDeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/MediaDeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..536468720 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/MediaDeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory.scala @@ -0,0 +1,130 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.MediaDeepRetrievalUserTweetANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.hydrator.MediaDeepRetrievalSignalTweetEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.MediaDeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams._ +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class MediaDeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory @Inject() ( + mediaDeepRetrievalTweetEmbeddingFeatureHydratorFactory: MediaDeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + mediaDeepRetrievalUserTweetANNCandidateSourceFactory: MediaDeepRetrievalUserTweetANNCandidateSourceFactory) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long], + enableFs: FSParam[Boolean], + maxCandidates: FSBoundedParam[Int] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): MediaDeepRetrievalTweetTweetSimilarityCandidatePipelineConfig[Query] = { + new MediaDeepRetrievalTweetTweetSimilarityCandidatePipelineConfig( + mediaDeepRetrievalTweetEmbeddingFeatureHydratorFactory, + signalFn, + mediaDeepRetrievalUserTweetANNCandidateSourceFactory, + identifierPrefix + CandidatePipelineConstants.MediaDeepRetrievalTweetTweetSimilarity, + MediaDeepRetrievalVectorDBCollectionName, + enableFs, + maxCandidates + ) + } +} + +class MediaDeepRetrievalTweetTweetSimilarityCandidatePipelineConfig[Query <: PipelineQuery]( + mediaDeepRetrievalTweetEmbeddingFeatureHydratorFactory: MediaDeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory, + signalFn: PipelineQuery => Seq[Long], + mediaDeepRetrievalUserTweetANNCandidateSourceFactory: MediaDeepRetrievalUserTweetANNCandidateSourceFactory, + identifierFullName: String, + vecDBCollectionName: FSParam[String], + enableMediaDeepRetrievalTweetAnn: FSParam[Boolean], + candidate: FSBoundedParam[Int], +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierFullName) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "MediaDeepRetrievalTweetANN", + param = enableMediaDeepRetrievalTweetAnn + ) + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq( + mediaDeepRetrievalTweetEmbeddingFeatureHydratorFactory.build(signalFn)) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + DRMultipleANNQuery + ] = { query => + val signals = query.features + .getOrElse(FeatureMap.empty) + .getOrElse( + MediaDeepRetrievalSignalTweetEmbeddingFeature, + MediaDeepRetrievalSignalTweetEmbeddingFeature.defaultValue) + + val keysOptSeq = signals.map { signal => + DRANNKey( + id = signal._1, + embedding = Some(signal._2), + collectionName = query.params(vecDBCollectionName), + maxCandidates = query.params(candidate) + ) + }.toSeq + DRMultipleANNQuery(keysOptSeq, false) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = mediaDeepRetrievalUserTweetANNCandidateSourceFactory.build(identifierFullName) + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..ec38e7a85 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfigFactory.scala @@ -0,0 +1,191 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.DRMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.MediaDeepRetrievalUserTweetANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.hydrator.MediaDeepRetrievalUserEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.MediaDeepRetrievalUserEmbeddingQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.hydrator.MediaEvergreenUserEmbeddingQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.hydrator.MediaPromotedCreatorUserEmbeddingQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaDeepRetrievalIsHighQualityEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaDeepRetrievalIsLowNegEngRatioEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaDeepRetrievalUserTweetANNEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaDeepRetrievalUserTweetANNMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaDeepRetrievalVectorDBCollectionName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaEvergreenDeepRetrievalUserTweetANNEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaEvergreenDeepRetrievalUserTweetANNMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaEvergreenDeepRetrievalVectorDBCollectionName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaPromotedCreatorDeepRetrievalUserTweetANNEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaPromotedCreatorDeepRetrievalUserTweetANNMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaPromotedCreatorDeepRetrievalVectorDBCollectionName +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfigFactory @Inject() ( + mediaDeepRetrievalUserEmbeddingFeatureHydrator: MediaDeepRetrievalUserEmbeddingQueryFeatureHydrator, + mediaDeepRetrievalUserTweetANNCandidateSourceFactory: MediaDeepRetrievalUserTweetANNCandidateSourceFactory) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + contentCategorySignalFn: Query => Option[Seq[Long]] = (q: Query) => None, + alerts: Seq[Alert] = Seq.empty + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfig[Query] = { + new MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfig( + mediaDeepRetrievalUserEmbeddingFeatureHydrator, + mediaDeepRetrievalUserTweetANNCandidateSourceFactory, + identifierPrefix + CandidatePipelineConstants.MediaDeepRetrievalUserTweetSimilarity, + MediaDeepRetrievalVectorDBCollectionName, + MediaDeepRetrievalUserTweetANNEnabled, + MediaDeepRetrievalUserTweetANNMaxCandidates, + contentCategorySignalFn, + alerts = alerts + ) + } +} + +@Singleton +class MediaEvergreenDeepRetrievalUserTweetSimilarityCandidatePipelineConfigFactory @Inject() ( + mediaEvergreenUserEmbeddingQueryFeatureHydrator: MediaEvergreenUserEmbeddingQueryFeatureHydrator, + mediaDeepRetrievalUserTweetANNCandidateSourceFactory: MediaDeepRetrievalUserTweetANNCandidateSourceFactory) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + alerts: Seq[Alert] = Seq.empty + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfig[Query] = { + new MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfig( + mediaEvergreenUserEmbeddingQueryFeatureHydrator, + mediaDeepRetrievalUserTweetANNCandidateSourceFactory, + identifierPrefix + CandidatePipelineConstants.MediaEvergreenDeepRetrievalUserTweetSimilarity, + MediaEvergreenDeepRetrievalVectorDBCollectionName, + MediaEvergreenDeepRetrievalUserTweetANNEnabled, + MediaEvergreenDeepRetrievalUserTweetANNMaxCandidates, + alerts = alerts + ) + } +} + +@Singleton +class MediaPromotedCreatorDeepRetrievalUserTweetSimilarityCandidatePipelineConfigFactory @Inject() ( + mediaPromotedCreatorUserEmbeddingQueryFeatureHydrator: MediaPromotedCreatorUserEmbeddingQueryFeatureHydrator, + mediaDeepRetrievalUserTweetANNCandidateSourceFactory: MediaDeepRetrievalUserTweetANNCandidateSourceFactory) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + alerts: Seq[Alert] = Seq.empty + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfig[Query] = { + new MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfig( + mediaPromotedCreatorUserEmbeddingQueryFeatureHydrator, + mediaDeepRetrievalUserTweetANNCandidateSourceFactory, + identifierPrefix + CandidatePipelineConstants.MediaPromotedCreatorDeepRetrievalUserTweetSimilarity, + MediaPromotedCreatorDeepRetrievalVectorDBCollectionName, + MediaPromotedCreatorDeepRetrievalUserTweetANNEnabled, + MediaPromotedCreatorDeepRetrievalUserTweetANNMaxCandidates, + alerts = alerts + ) + } +} + +class MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfig[Query <: PipelineQuery]( + mediaDeepRetrievalUserEmbeddingFeatureHydrator: MediaDeepRetrievalUserEmbeddingQueryFeatureHydrator, + mediaDeepRetrievalUserTweetANNCandidateSourceFactory: MediaDeepRetrievalUserTweetANNCandidateSourceFactory, + identifierFullName: String, + vecDBCollectionName: FSParam[String], + enableMediaDeepRetrievalAnn: FSParam[Boolean], + candidate: FSBoundedParam[Int], + contentCategorySignalFn: Query => Option[Seq[Long]] = (q: Query) => None, + override val alerts: Seq[Alert] +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + DRMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier(identifierFullName) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "MediaDeepRetrievalUserTweetANN", + param = enableMediaDeepRetrievalAnn + ) + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq( + mediaDeepRetrievalUserEmbeddingFeatureHydrator) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + DRMultipleANNQuery + ] = { query => + val defaultKey = DRANNKey( + id = query.getRequiredUserId, + embedding = query.features.flatMap(_.getOrElse(MediaDeepRetrievalUserEmbeddingFeature, None)), + collectionName = query.params(vecDBCollectionName), + maxCandidates = query.params(candidate), + isHighQuality = + if (query.params(MediaDeepRetrievalIsHighQualityEnabled)) Some(true) else None, + isLowNegEngRatio = + if (query.params(MediaDeepRetrievalIsLowNegEngRatioEnabled)) Some(false) else None + ) + val keysOptSeq = contentCategorySignalFn(query).map(_.map(_.toString)).map { categories => + categories.map { category => + DRANNKey( + id = query.getRequiredUserId, + embedding = + query.features.flatMap(_.getOrElse(MediaDeepRetrievalUserEmbeddingFeature, None)), + collectionName = query.params(vecDBCollectionName), + maxCandidates = query.params(candidate), + category = Some(category), + isHighQuality = + if (query.params(MediaDeepRetrievalIsHighQualityEnabled)) Some(true) else None, + isLowNegEngRatio = + if (query.params(MediaDeepRetrievalIsLowNegEngRatioEnabled)) Some(false) else None + ) + } + } + val keys = keysOptSeq.getOrElse(Seq(defaultKey)) + DRMultipleANNQuery(keys, false) + } + + override def candidateSource: CandidateSource[ + DRMultipleANNQuery, + TweetMixerCandidate + ] = mediaDeepRetrievalUserTweetANNCandidateSourceFactory.build(identifierFullName) + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => TweetCandidate(id = candidate.tweetId) } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PinnedTweetRelatedCreatorPipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PinnedTweetRelatedCreatorPipelineConfigFactory.scala new file mode 100644 index 000000000..98c1602bd --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PinnedTweetRelatedCreatorPipelineConfigFactory.scala @@ -0,0 +1,89 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.candidate_source.timeline_service.TimelineServiceTweetCandidateSource +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelineservice.{thriftscala => tlsthrift} +import com.twitter.tweet_mixer.functional_component.transformer.TimelineQueryTransformer +import com.twitter.tweet_mixer.functional_component.transformer.TweetFeatureTimelineServiceTransformer +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableRelatedCreatorParam +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class PinnedTweetRelatedCreatorPipelineConfigFactory @Inject() ( + timelineServiceTweetCandidateSource: TimelineServiceTweetCandidateSource) { + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): PinnedTweetRelatedCreatorPipelineConfig[Query] = { + new PinnedTweetRelatedCreatorPipelineConfig( + identifierPrefix, + signalFn, + timelineServiceTweetCandidateSource) + } +} + +class PinnedTweetRelatedCreatorPipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long], + timelineServiceTweetCandidateSource: TimelineServiceTweetCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + tlsthrift.TimelineQuery, + tlsthrift.Tweet, + TweetCandidate + ] { + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "EnableRelatedCreatorParam", + param = EnableRelatedCreatorParam + ) + ) + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.RelatedCreator) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + tlsthrift.TimelineQuery + ] = TimelineQueryTransformer(tlsthrift.TimelineType.Media, signalFn, Some(true)) + override def candidateSource: BaseCandidateSource[tlsthrift.TimelineQuery, tlsthrift.Tweet] = + timelineServiceTweetCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[tlsthrift.Tweet] + ] = Seq(TweetFeatureTimelineServiceTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + tlsthrift.Tweet, + TweetCandidate + ] = { candidate => + TweetCandidate( + id = candidate._1 + ) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopGrokTopicTweetsCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopGrokTopicTweetsCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..6ff56db8e --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopGrokTopicTweetsCandidatePipelineConfigFactory.scala @@ -0,0 +1,73 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.candidate_source.popular_grok_topic_tweets.GrokTopicTweetsQuery +import com.twitter.tweet_mixer.candidate_source.popular_grok_topic_tweets.PopGrokTopicTweetsCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.GrokTopicTweetsQueryTransformer +import com.twitter.tweet_mixer.param.PopGrokTopicTweetsParams +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class PopGrokTopicTweetsCandidatePipelineConfigFactory @Inject() ( + popGrokTopicTweetsCandidateSource: PopGrokTopicTweetsCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): PopGrokTopicTweetsCandidatePipelineConfig[Query] = { + new PopGrokTopicTweetsCandidatePipelineConfig( + identifierPrefix, + popGrokTopicTweetsCandidateSource + ) + } +} + +class PopGrokTopicTweetsCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + popGrokTopicTweetsCandidateSource: PopGrokTopicTweetsCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + GrokTopicTweetsQuery, + TweetCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.PopGrokTopicTweets) + + override val supportedClientParam: Option[FSParam[Boolean]] = + Some(PopGrokTopicTweetsParams.PopGrokTopicTweetsEnable) + + override val queryTransformer: CandidatePipelineQueryTransformer[Query, GrokTopicTweetsQuery] = + GrokTopicTweetsQueryTransformer(PopGrokTopicTweetsParams.MaxNumCandidates) + + override def candidateSource: CandidateSource[ + GrokTopicTweetsQuery, + TweetCandidate + ] = popGrokTopicTweetsCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetCandidate, + TweetCandidate + ] = { sourceResult => sourceResult } + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert(warnThreshold = 80, criticalThreshold = 90), + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopularGeoTweetsCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopularGeoTweetsCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..50c6b89a1 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopularGeoTweetsCandidatePipelineConfigFactory.scala @@ -0,0 +1,81 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet +import com.twitter.tweet_mixer.candidate_source.popular_geo_tweets.PopularGeoTweetsCandidateSource +import com.twitter.tweet_mixer.candidate_source.popular_geo_tweets.TripStratoGeoQuery +import com.twitter.tweet_mixer.functional_component.transformer.TripStratoGeoQueryTransformer +import com.twitter.tweet_mixer.functional_component.transformer.TripTweetFeatureTransformer +import com.twitter.tweet_mixer.param.PopularGeoTweetsParams +import com.twitter.tweet_mixer.param.PopularGeoTweetsParams.PopularGeoTweetsEnable +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class PopularGeoTweetsCandidatePipelineConfigFactory @Inject() ( + popularGeoTweetsCandidateSource: PopularGeoTweetsCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): PopularGeoTweetsCandidatePipelineConfig[Query] = { + new PopularGeoTweetsCandidatePipelineConfig(identifierPrefix, popularGeoTweetsCandidateSource) + } +} + +class PopularGeoTweetsCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + popularGeoTweetsCandidateSource: PopularGeoTweetsCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + TripStratoGeoQuery, + TripTweet, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.PopularGeoTweets) + + override val supportedClientParam: Option[FSParam[Boolean]] = Some(PopularGeoTweetsEnable) + + override val queryTransformer: CandidatePipelineQueryTransformer[Query, TripStratoGeoQuery] = + TripStratoGeoQueryTransformer( + geoSourceIdsParam = PopularGeoTweetsParams.GeoSourceIds, + maxTweetsPerDomainParam = PopularGeoTweetsParams.MaxNumCandidatesPerTripSource, + maxPopGeoTweetsParam = PopularGeoTweetsParams.MaxNumPopGeoCandidates + ) + + override def candidateSource: CandidateSource[ + TripStratoGeoQuery, + TripTweet + ] = popularGeoTweetsCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TripTweet, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TripTweet] + ] = Seq(TripTweetFeatureTransformer) + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopularTopicTweetsCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopularTopicTweetsCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..54abaf0c4 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/PopularTopicTweetsCandidatePipelineConfigFactory.scala @@ -0,0 +1,84 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet +import com.twitter.tweet_mixer.candidate_source.popular_topic_tweets.PopularTopicTweetsCandidateSource +import com.twitter.tweet_mixer.candidate_source.popular_topic_tweets.TripStratoTopicQuery +import com.twitter.tweet_mixer.functional_component.transformer.TripStratoTopicQueryTransformer +import com.twitter.tweet_mixer.functional_component.transformer.TripTweetFeatureTransformer +import com.twitter.tweet_mixer.param.PopularTopicTweetsParams +import com.twitter.tweet_mixer.param.PopularTopicTweetsParams.PopularTopicTweetsEnable +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class PopularTopicTweetsCandidatePipelineConfigFactory @Inject() ( + popularTopicTweetsCandidateSource: PopularTopicTweetsCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): PopularTopicTweetsCandidatePipelineConfig[Query] = { + new PopularTopicTweetsCandidatePipelineConfig( + identifierPrefix, + popularTopicTweetsCandidateSource) + } +} + +class PopularTopicTweetsCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + popularTopicTweetsCandidateSource: PopularTopicTweetsCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + TripStratoTopicQuery, + TripTweet, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.PopularTopicTweets) + + override val supportedClientParam: Option[FSParam[Boolean]] = Some(PopularTopicTweetsEnable) + + override val queryTransformer: CandidatePipelineQueryTransformer[Query, TripStratoTopicQuery] = + TripStratoTopicQueryTransformer( + sourceIdsParam = PopularTopicTweetsParams.SourceIds, + maxTweetsPerDomainParam = PopularTopicTweetsParams.MaxNumCandidatesPerTripSource, + maxTweetsParam = PopularTopicTweetsParams.MaxNumCandidates, + popTopicIdsParam = PopularTopicTweetsParams.PopTopicIds + ) + + override def candidateSource: CandidateSource[ + TripStratoTopicQuery, + TripTweet + ] = popularTopicTweetsCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TripTweet, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TripTweet] + ] = Seq(TripTweetFeatureTransformer) + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert(warnThreshold = 80, criticalThreshold = 90), + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/QigSearchHistoryTweetsCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/QigSearchHistoryTweetsCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..03f0946f2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/QigSearchHistoryTweetsCandidatePipelineConfigFactory.scala @@ -0,0 +1,108 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.filter.ParamGatedFilter +import com.twitter.product_mixer.component_library.filter.TweetLanguageFilter +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.tweet_mixer.functional_component.gate.AllowNonEmptySearchHistoryUserGate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.qig_service.QigServiceBatchTweetCandidateSource +import com.twitter.search.query_interaction_graph.service.{thriftscala => t} +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.candidate_source.qig_service.QigTweetCandidate +import com.twitter.tweet_mixer.feature.LanguageCodeFeature +import com.twitter.tweet_mixer.functional_component.transformer.QigBatchQueryTransformer +import com.twitter.tweet_mixer.functional_component.transformer.QigTweetCandidateFeatureTransformer +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.QigSearchHistoryCandidateSourceEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.QigSearchHistoryTweetsEnableLanguageFilter +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.QigSearchHistoryTweetsEnabled +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class QigSearchHistoryTweetsCandidatePipelineConfigFactory @Inject() ( + qigServiceBatchTweetCandidateSource: QigServiceBatchTweetCandidateSource, +) { + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[String] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): QigSearchHistoryTweetsCandidatePipelineConfig[Query] = { + new QigSearchHistoryTweetsCandidatePipelineConfig( + identifierPrefix, + signalFn, + qigServiceBatchTweetCandidateSource, + ) + } +} + +class QigSearchHistoryTweetsCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[String], + qigServiceBatchTweetCandidateSource: QigServiceBatchTweetCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + Seq[t.QigRequest], + QigTweetCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.QigSearchHistoryTweets + ) + + override val supportedClientParam: Option[FSParam[Boolean]] = Some(QigSearchHistoryTweetsEnabled) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + AllowNonEmptySearchHistoryUserGate(signalFn), + ParamGate( + name = "QigSearchHistoryCandidateSourceEnabled", + param = QigSearchHistoryCandidateSourceEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + Seq[t.QigRequest] + ] = QigBatchQueryTransformer(signalFn) + + override val candidateSource: CandidateSource[ + Seq[t.QigRequest], + QigTweetCandidate + ] = + qigServiceBatchTweetCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[QigTweetCandidate] + ] = Seq( + QigTweetCandidateFeatureTransformer + ) + + override def filters: Seq[Filter[Query, TweetCandidate]] = Seq( + ParamGatedFilter( + QigSearchHistoryTweetsEnableLanguageFilter, + TweetLanguageFilter(LanguageCodeFeature)), + ) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + QigTweetCandidate, + TweetCandidate + ] = { sourceResult => + TweetCandidate( + id = sourceResult.tweetCandidate.tweetId + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SemanticVideoCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SemanticVideoCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..08d6afe77 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SemanticVideoCandidatePipelineConfigFactory.scala @@ -0,0 +1,91 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.EvergreenVideosSearchByTweetQuery +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.SemanticVideoCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.EvergreenVideosResponseFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.TwitterClipV0LongVideoQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.SemanticVideoCandidatePipelineEnabled +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SemanticVideoCandidatePipelineConfigFactory @Inject() ( + semanticVideoCandidateSource: SemanticVideoCandidateSource) { + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): SemanticVideoCandidatePipelineConfig[Query] = { + val identifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.SemanticVideo) + new SemanticVideoCandidatePipelineConfig( + identifier, + semanticVideoCandidateSource, + TwitterClipV0LongVideoQueryTransformer(signalFn, identifier) + ) + } +} + +class SemanticVideoCandidatePipelineConfig[Query <: PipelineQuery]( + override val identifier: CandidatePipelineIdentifier, + semanticVideoCandidateSource: CandidateSource[ + EvergreenVideosSearchByTweetQuery, + TweetMixerCandidate, + ], + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + EvergreenVideosSearchByTweetQuery + ] +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + EvergreenVideosSearchByTweetQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "SemanticVideoCandidatePipelineEnabled", + param = SemanticVideoCandidatePipelineEnabled + ) + ) + + override def candidateSource: CandidateSource[ + EvergreenVideosSearchByTweetQuery, + TweetMixerCandidate, + ] = semanticVideoCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { tweet => + TweetCandidate( + id = tweet.tweetId + ) + } + + override val featuresFromCandidateSourceTransformers = Seq( + EvergreenVideosResponseFeatureTransformer) + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert(warnThreshold = 60, criticalThreshold = 95) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersInterestedInCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersInterestedInCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..696339aee --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersInterestedInCandidatePipelineConfigFactory.scala @@ -0,0 +1,108 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.simclusters_v2.thriftscala.EmbeddingType.FollowBasedUserInterestedIn +import com.twitter.simclusters_v2.thriftscala.EmbeddingType.UnfilteredUserInterestedIn +import com.twitter.simclusters_v2.thriftscala.InternalId +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SANNQuery +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SimClustersAnnCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.SANNQueryTransformer +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.SimClustersANNParams._ +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.SimclustersInterestedInEnabled +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SimclustersInterestedInCandidatePipelineConfigFactory @Inject() ( + simClustersAnnCandidateSource: SimClustersAnnCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): SimclustersInterestedInCandidatePipelineConfig[Query] = { + new SimclustersInterestedInCandidatePipelineConfig( + identifierPrefix, + simClustersAnnCandidateSource) + } +} + +class SimclustersInterestedInCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + simClustersAnnCandidateSource: SimClustersAnnCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + SANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.SimClustersInterestedIn) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "SimclustersInterestedInEnabled", + param = SimclustersInterestedInEnabled + ) + ) + + private val signalFn: PipelineQuery => Seq[InternalId] = { query => + Seq(InternalId.UserId(query.getRequiredUserId)) + } + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + SANNQuery + ] = { query => + val embeddingTypes = + if (query.params(EnableAdditionalInterestedInEmbeddingTypesParam)) + Seq(FollowBasedUserInterestedIn, UnfilteredUserInterestedIn) + else Seq(UnfilteredUserInterestedIn) + + SANNQueryTransformer( + TransformerIdentifier("SANNQueryInterestedIn"), + InterestedInClusterParamMap, + signalFn, + embeddingTypes, + InterestedInMinScoreParam, + Some(InterestedInMaxCandidatesParam) + ).transform(query) + } + + override def candidateSource: CandidateSource[ + SANNQuery, + TweetMixerCandidate + ] = simClustersAnnCandidateSource + + override val featuresFromCandidateSourceTransformers = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersProducerBasedCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersProducerBasedCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..32a54b1e3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersProducerBasedCandidatePipelineConfigFactory.scala @@ -0,0 +1,103 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.simclusters_v2.thriftscala.EmbeddingType.FavBasedProducer +import com.twitter.simclusters_v2.thriftscala.InternalId +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SANNQuery +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SimClustersAnnCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.SANNQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.SimClustersANNParams.EnableProducerBasedMaxCandidatesParam +import com.twitter.tweet_mixer.param.SimClustersANNParams.ProducerBasedClusterParamMap +import com.twitter.tweet_mixer.param.SimClustersANNParams.ProducerBasedMaxCandidatesParam +import com.twitter.tweet_mixer.param.SimClustersANNParams.ProducerBasedMinScoreParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.SimclustersProducerBasedEnabled +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SimclustersProducerBasedCandidatePipelineConfigFactory @Inject() ( + simClustersAnnCandidateSource: SimClustersAnnCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long], + alerts: Seq[Alert] = Seq.empty + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): SimclustersProducerBasedCandidatePipelineConfig[Query] = { + new SimclustersProducerBasedCandidatePipelineConfig( + identifierPrefix, + signalFn, + simClustersAnnCandidateSource, + alerts + ) + } +} + +class SimclustersProducerBasedCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long], + simClustersAnnCandidateSource: SimClustersAnnCandidateSource, + override val alerts: Seq[Alert] +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + SANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.SimClustersProducerBased) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "SimclustersProducerBasedEnabled", + param = SimclustersProducerBasedEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + SANNQuery + ] = { query => + SANNQueryTransformer( + TransformerIdentifier("SANNQueryProducerBased"), + ProducerBasedClusterParamMap, + query => signalFn(query).map(InternalId.UserId(_)), + Seq(FavBasedProducer), + ProducerBasedMinScoreParam, + maxInterestedInCandidatesParam = if (query.params(EnableProducerBasedMaxCandidatesParam)) { + Some(ProducerBasedMaxCandidatesParam) + } else { None } + ).transform(query) + } + + override def candidateSource: CandidateSource[ + SANNQuery, + TweetMixerCandidate + ] = simClustersAnnCandidateSource + + override val featuresFromCandidateSourceTransformers = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersPromotedCreatorCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersPromotedCreatorCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..648fad9f0 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersPromotedCreatorCandidatePipelineConfigFactory.scala @@ -0,0 +1,105 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.simclusters_v2.thriftscala.EmbeddingType.LogFavLongestL2EmbeddingTweet +import com.twitter.simclusters_v2.thriftscala.InternalId +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SANNQuery +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SimClustersAnnCandidateSource +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SimclusterPromotedCreatorAnnCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.SANNQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.SimClustersANNParams.TweetBasedClusterParamMap +import com.twitter.tweet_mixer.param.SimClustersANNParams.TweetBasedMinScoreParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.SimclustersPromotedCreatorEnabled +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SimclustersPromotedCreatorCandidatePipelineConfigFactory @Inject() ( + simClustersAnnCandidateSource: SimclusterPromotedCreatorAnnCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): SimclustersPromotedCreatorCandidatePipelineConfig[Query] = { + new SimclustersPromotedCreatorCandidatePipelineConfig( + identifierPrefix, + signalsFn = signalsFn, + simClustersAnnCandidateSource) + } +} + +class SimclustersPromotedCreatorCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long], + simClustersAnnCandidateSource: SimClustersAnnCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + SANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.SimClustersPromotedCreator) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "SimclustersPromotedCreatorEnabled", + param = SimclustersPromotedCreatorEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + SANNQuery + ] = { + SANNQueryTransformer( + TransformerIdentifier("SANNQueryPromotedCreator"), + TweetBasedClusterParamMap, + query => signalFnSelector(query), + Seq(LogFavLongestL2EmbeddingTweet), + TweetBasedMinScoreParam + ) + } + + private def signalFnSelector(query: PipelineQuery): Seq[InternalId.TweetId] = { + signalsFn(query).map(InternalId.TweetId(_)) + } + + override def candidateSource: CandidateSource[ + SANNQuery, + TweetMixerCandidate + ] = simClustersAnnCandidateSource + + override val featuresFromCandidateSourceTransformers = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersTweetBasedCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersTweetBasedCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..daf8c9990 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SimclustersTweetBasedCandidatePipelineConfigFactory.scala @@ -0,0 +1,102 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.simclusters_v2.thriftscala.EmbeddingType.LogFavLongestL2EmbeddingTweet +import com.twitter.simclusters_v2.thriftscala.InternalId +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SANNQuery +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SimClustersAnnCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.SANNQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.SimClustersANNParams.TweetBasedClusterParamMap +import com.twitter.tweet_mixer.param.SimClustersANNParams.TweetBasedMinScoreParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.SimclustersTweetBasedEnabled +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SimclustersTweetBasedCandidatePipelineConfigFactory @Inject() ( + simClustersAnnCandidateSource: SimClustersAnnCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long], + alerts: Seq[Alert] = Seq.empty + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): SimclustersTweetBasedCandidatePipelineConfig[Query] = { + new SimclustersTweetBasedCandidatePipelineConfig( + identifierPrefix, + signalsFn = signalsFn, + simClustersAnnCandidateSource, + alerts + ) + } +} + +class SimclustersTweetBasedCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long], + simClustersAnnCandidateSource: SimClustersAnnCandidateSource, + override val alerts: Seq[Alert] +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + SANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.SimClustersTweetBased) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "SimclustersTweetBasedEnabled", + param = SimclustersTweetBasedEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + SANNQuery + ] = { + SANNQueryTransformer( + TransformerIdentifier("SANNQueryTweetBased"), + TweetBasedClusterParamMap, + query => signalFnSelector(query), + Seq(LogFavLongestL2EmbeddingTweet), + TweetBasedMinScoreParam + ) + } + + private def signalFnSelector(query: PipelineQuery): Seq[InternalId.TweetId] = { + signalsFn(query).map(InternalId.TweetId(_)) + } + + override def candidateSource: CandidateSource[ + SANNQuery, + TweetMixerCandidate + ] = simClustersAnnCandidateSource + + override val featuresFromCandidateSourceTransformers = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SkitTopicTweetsCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SkitTopicTweetsCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..28b0456a5 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/SkitTopicTweetsCandidatePipelineConfigFactory.scala @@ -0,0 +1,86 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.candidate_source.topic_tweets.SkitTopicTweetsCandidateSource +import com.twitter.tweet_mixer.candidate_source.topic_tweets.SkitTopicTweetsQuery +import com.twitter.tweet_mixer.functional_component.transformer.TopicTweetFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SkitTopicTweetsCandidatePipelineConfigFactory @Inject() ( + skitTopicTweetsCandidateSource: SkitTopicTweetsCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + enabledParam: FSParam[Boolean], + identifierString: String, + queryTransformer: CandidatePipelineQueryTransformer[Query, SkitTopicTweetsQuery] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): SkitTopicTweetsCandidatePipelineConfig[Query] = { + new SkitTopicTweetsCandidatePipelineConfig( + identifierPrefix = identifierPrefix, + enabledParam = enabledParam, + identifierString = identifierString, + candidateSource = skitTopicTweetsCandidateSource, + queryTransformer = queryTransformer, + ) + } +} + +class SkitTopicTweetsCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + enabledParam: FSParam[Boolean], + identifierString: String, + override val candidateSource: CandidateSource[ + SkitTopicTweetsQuery, + TweetMixerCandidate + ], + override val queryTransformer: CandidatePipelineQueryTransformer[Query, SkitTopicTweetsQuery] +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + SkitTopicTweetsQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + identifierString) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = s"${identifierString}Enabled", + param = enabledParam + ) + ) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer, TopicTweetFeatureTransformer) + + override val alerts = Seq( + defaultSuccessRateAlert(), + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TrendsVideoCandidatePipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TrendsVideoCandidatePipelineConfig.scala new file mode 100644 index 000000000..066b2271f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TrendsVideoCandidatePipelineConfig.scala @@ -0,0 +1,69 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.component_library.gate.DefinedCountryCodeGate +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.strato.generated.client.trendsai.media.TopCountryVideosClientColumn +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.candidate_source.trends.TrendsVideoCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.CandidateSourceParams.TrendsVideoEnabled +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.BusinessHours +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.CoreProductGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TrendsVideoCandidatePipelineConfig @Inject() ( + trendsVideoCandidateSource: TrendsVideoCandidateSource) + extends CandidatePipelineConfig[ + PipelineQuery, + TopCountryVideosClientColumn.Key, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier(CandidatePipelineConstants.TrendsVideo) + + override val supportedClientParam: Option[FSParam[Boolean]] = Some(TrendsVideoEnabled) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + DefinedCountryCodeGate + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + TopCountryVideosClientColumn.Key, + ] = { query => query.getCountryCode.get } + + override def candidateSource: CandidateSource[ + TopCountryVideosClientColumn.Key, + TweetMixerCandidate + ] = trendsVideoCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => TweetCandidate(id = candidate.tweetId) } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(threshold = 95, notificationType = BusinessHours)(CoreProductGroupMap), + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwHINRebuildTweetSimilarityCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwHINRebuildTweetSimilarityCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..43bb74595 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwHINRebuildTweetSimilarityCandidatePipelineConfigFactory.scala @@ -0,0 +1,100 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.twhin_ann.TwHINRebuildANNKey +import com.twitter.tweet_mixer.candidate_source.twhin_ann.TwHINRebuildANNCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.TwhinRebuildTweetSimilarityEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.TwhinRebuildTweetSimilarityDatasetParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.TwhinRebuildTweetSimilarityDatasetEnum +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.TwhinRebuildTweetSimilarityMaxCandidates +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TwHINRebuildTweetSimilarityCandidatePipelineConfigFactory @Inject() ( + twHINANNCandidateSource: TwHINRebuildANNCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): TwHINRebuildTweetSimilarityCandidatePipelineConfig[Query] = { + new TwHINRebuildTweetSimilarityCandidatePipelineConfig( + twHINANNCandidateSource, + signalFn, + identifierPrefix + ) + } +} + +class TwHINRebuildTweetSimilarityCandidatePipelineConfig[Query <: PipelineQuery]( + twHINANNCandidateSource: TwHINRebuildANNCandidateSource, + signalFn: PipelineQuery => Seq[Long], + identifierPrefix: String +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + Seq[TwHINRebuildANNKey], + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.TwhinRebuildTweetSimilarity) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "TwHINRebuildTweetSimilarity", + param = TwhinRebuildTweetSimilarityEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + Seq[TwHINRebuildANNKey] + ] = { query => + val dataset = query.params(TwhinRebuildTweetSimilarityDatasetParam) + val versionId = TwhinRebuildTweetSimilarityDatasetEnum.enumToTwhinRebuildVersionIdMap(dataset) + val maxCandidates = query.params(TwhinRebuildTweetSimilarityMaxCandidates) + signalFn(query).map(tweetId => + TwHINRebuildANNKey(tweetId, dataset.toString, versionId, maxCandidates)) + } + + override def candidateSource: CandidateSource[ + Seq[TwHINRebuildANNKey], + TweetMixerCandidate + ] = twHINANNCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwHINTweetSimilarityCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwHINTweetSimilarityCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..7e81abc58 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwHINTweetSimilarityCandidatePipelineConfigFactory.scala @@ -0,0 +1,93 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.simclusters_v2.common.TweetId +import com.twitter.tweet_mixer.candidate_source.twhin_ann.TwHINANNCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.TwhinTweetSimilarityEnabled +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TwHINTweetSimilarityCandidatePipelineConfigFactory @Inject() ( + twHINANNCandidateSource: TwHINANNCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): TwHINTweetSimilarityCandidatePipelineConfig[Query] = { + new TwHINTweetSimilarityCandidatePipelineConfig( + twHINANNCandidateSource, + signalFn, + identifierPrefix + ) + } +} + +class TwHINTweetSimilarityCandidatePipelineConfig[Query <: PipelineQuery]( + twHINANNCandidateSource: TwHINANNCandidateSource, + signalFn: PipelineQuery => Seq[Long], + identifierPrefix: String +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + Seq[TweetId], + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.TwhinTweetSimilarity) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "TwHINTweetSimilarity", + param = TwhinTweetSimilarityEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + Seq[TweetId] + ] = { query => + signalFn(query) + } + + override def candidateSource: CandidateSource[ + Seq[TweetId], + TweetMixerCandidate + ] = twHINANNCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwhinConsumerBasedCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwhinConsumerBasedCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..17eb94d99 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwhinConsumerBasedCandidatePipelineConfigFactory.scala @@ -0,0 +1,105 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.ann.common.CosineDistance +import com.twitter.ann.common.NeighborWithDistanceWithSeed +import com.twitter.ann.common.QueryableById +import com.twitter.ann.hnsw.HnswParams +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.product_mixer.component_library.candidate_source.ann.AnnCandidateSource +import com.twitter.product_mixer.component_library.candidate_source.ann.AnnIdQuery +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.functional_component.transformer.AnnCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.TwhinConsumerBasedEnabled +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class TwhinConsumerBasedCandidatePipelineConfigFactory @Inject() ( + @Named(ModuleNames.ConsumerBasedTwHINAnnQueryableById) + twhinConsumerBasedAnnQueryableById: QueryableById[Long, Long, HnswParams, CosineDistance]) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): TwhinConsumerBasedCandidatePipelineConfig[Query] = { + new TwhinConsumerBasedCandidatePipelineConfig( + identifierPrefix, + twhinConsumerBasedAnnQueryableById + ) + } +} + +class TwhinConsumerBasedCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + @Named(ModuleNames.ConsumerBasedTwHINAnnQueryableById) + twhinConsumerBasedAnnQueryableById: QueryableById[Long, Long, HnswParams, CosineDistance] +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + AnnIdQuery[Long, HnswParams], + NeighborWithDistanceWithSeed[Long, Long, CosineDistance], + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.TwhinConsumerBased) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "TwhinConsumerBasedEnabled", + param = TwhinConsumerBasedEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + AnnIdQuery[Long, HnswParams] + ] = { query => + AnnIdQuery(Seq(query.getRequiredUserId), numOfNeighbors = 200, HnswParams(800)) + } + + override def candidateSource: CandidateSource[ + AnnIdQuery[Long, HnswParams], + NeighborWithDistanceWithSeed[Long, Long, CosineDistance] + ] = + new AnnCandidateSource( + twhinConsumerBasedAnnQueryableById, + batchSize = 20, + timeoutPerRequest = 200.millis, + identifier = CandidateSourceIdentifier("TwhinConsumerBasedAnn")) + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[NeighborWithDistanceWithSeed[Long, Long, CosineDistance]] + ] = Seq(AnnCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + NeighborWithDistanceWithSeed[Long, Long, CosineDistance], + TweetCandidate + ] = { neighbor => + TweetCandidate( + id = neighbor.neighbor + ) + } + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwhinUserTweetSimilarityCandidatePipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwhinUserTweetSimilarityCandidatePipelineConfig.scala new file mode 100644 index 000000000..18604e163 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwhinUserTweetSimilarityCandidatePipelineConfig.scala @@ -0,0 +1,94 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.EmbeddingANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.EmbeddingMultipleANNQuery +import com.twitter.tweet_mixer.candidate_source.ndr_ann.EmbeddingANNCandidateSourceFactory +import com.twitter.tweet_mixer.functional_component.hydrator.TwhinUserPositiveEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.TwhinUserPositiveEmbeddingQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.TwhinRebuildUserTweetSimilarityEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.TwhinRebuildUserTweetSimilarityMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.TwhinRebuildUserTweetVectorDBName +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TwhinUserTweetSimilarityCandidatePipelineConfig @Inject() ( + twhinUserPositiveEmbeddingQueryFeatureHydrator: TwhinUserPositiveEmbeddingQueryFeatureHydrator, + embeddingANNCandidateSourceFactory: EmbeddingANNCandidateSourceFactory, + identifierPrefix: String) + extends CandidatePipelineConfig[ + PipelineQuery, + EmbeddingMultipleANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.TwhinRebuildUserTweetSimilarity) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "TwhinUserTweetANN", + param = TwhinRebuildUserTweetSimilarityEnabled + ) + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + twhinUserPositiveEmbeddingQueryFeatureHydrator, + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + EmbeddingMultipleANNQuery + ] = { query => + val defaultKey = EmbeddingANNKey( + id = query.getRequiredUserId, + embedding = query.features + .flatMap(_.getOrElse(TwhinUserPositiveEmbeddingFeature, None)), + collectionName = query.params(TwhinRebuildUserTweetVectorDBName), + maxCandidates = query.params(TwhinRebuildUserTweetSimilarityMaxCandidates), + ) + + EmbeddingMultipleANNQuery(Seq(defaultKey), false) + } + + override def candidateSource: CandidateSource[ + EmbeddingMultipleANNQuery, + TweetMixerCandidate + ] = embeddingANNCandidateSourceFactory.build( + identifierPrefix + CandidatePipelineConstants.TwhinRebuildUserTweetSimilarity) + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert()(ForYouGroupMap), + defaultEmptyResponseRateAlert()(ForYouGroupMap) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwitterClipV0LongVideoCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwitterClipV0LongVideoCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..4cbc86820 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwitterClipV0LongVideoCandidatePipelineConfigFactory.scala @@ -0,0 +1,89 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.EvergreenVideosSearchByTweetQuery +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.TwitterClipV0LongVideoCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.EvergreenVideosResponseFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.TwitterClipV0LongVideoQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.TwitterClipV0LongVideoCandidatePipelineEnabled +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TwitterClipV0LongVideoCandidatePipelineConfigFactory @Inject() ( + twitterClipV0LongVideoCandidateSource: TwitterClipV0LongVideoCandidateSource) { + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): TwitterClipV0LongVideoCandidatePipelineConfig[Query] = { + val identifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.TwitterClipV0Long) + new TwitterClipV0LongVideoCandidatePipelineConfig( + identifier, + twitterClipV0LongVideoCandidateSource, + TwitterClipV0LongVideoQueryTransformer(signalFn, identifier) + ) + } +} + +class TwitterClipV0LongVideoCandidatePipelineConfig[Query <: PipelineQuery]( + override val identifier: CandidatePipelineIdentifier, + twitterClipV0LongVideoCandidateSource: CandidateSource[ + EvergreenVideosSearchByTweetQuery, + TweetMixerCandidate, + ], + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + EvergreenVideosSearchByTweetQuery + ] +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + EvergreenVideosSearchByTweetQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "TwitterClipV0LongVideoCandidatePipelineEnabled", + param = TwitterClipV0LongVideoCandidatePipelineEnabled + ) + ) + + override def candidateSource: CandidateSource[ + EvergreenVideosSearchByTweetQuery, + TweetMixerCandidate, + ] = twitterClipV0LongVideoCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { tweet => + TweetCandidate( + id = tweet.tweetId + ) + } + + override val featuresFromCandidateSourceTransformers = Seq( + EvergreenVideosResponseFeatureTransformer) + + override val alerts = Seq( + defaultSuccessRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwitterClipV0ShortVideoCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwitterClipV0ShortVideoCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..3d32d8ebc --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/TwitterClipV0ShortVideoCandidatePipelineConfigFactory.scala @@ -0,0 +1,91 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.EvergreenVideosSearchByTweetQuery +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.TwitterClipV0ShortVideoCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.EvergreenVideosResponseFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.TwitterClipV0ShortVideoQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.TwitterClipV0ShortVideoCandidatePipelineEnabled +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TwitterClipV0ShortVideoCandidatePipelineConfigFactory @Inject() ( + twitterClipV0ShortVideoCandidateSource: TwitterClipV0ShortVideoCandidateSource) { + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): TwitterClipV0ShortVideoCandidatePipelineConfig[Query] = { + val identifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.TwitterClipV0Short) + new TwitterClipV0ShortVideoCandidatePipelineConfig( + identifier, + twitterClipV0ShortVideoCandidateSource, + TwitterClipV0ShortVideoQueryTransformer(signalFn, identifier) + ) + } +} + +class TwitterClipV0ShortVideoCandidatePipelineConfig[Query <: PipelineQuery]( + override val identifier: CandidatePipelineIdentifier, + twitterClipV0ShortVideoCandidateSource: CandidateSource[ + EvergreenVideosSearchByTweetQuery, + TweetMixerCandidate, + ], + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + EvergreenVideosSearchByTweetQuery + ] +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + EvergreenVideosSearchByTweetQuery, + TweetMixerCandidate, + TweetCandidate + ] { + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "TwitterClipV0ShortVideoCandidatePipelineEnabled", + param = TwitterClipV0ShortVideoCandidatePipelineEnabled + ) + ) + + override def candidateSource: CandidateSource[ + EvergreenVideosSearchByTweetQuery, + TweetMixerCandidate, + ] = twitterClipV0ShortVideoCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { tweet => + TweetCandidate( + id = tweet.tweetId + ) + } + + override val featuresFromCandidateSourceTransformers = Seq( + EvergreenVideosResponseFeatureTransformer) + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTEGCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTEGCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..dc8d3a5cb --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTEGCandidatePipelineConfigFactory.scala @@ -0,0 +1,85 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.candidate_source.uteg.UtegCandidateSource +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.HasExcludedIds +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} +import com.twitter.tweet_mixer.functional_component.transformer.UtegQueryTransformer +import com.twitter.tweet_mixer.functional_component.transformer.UtegResponseFeatureTransformer +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UTEGEnabled +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UTEGCandidatePipelineConfigFactory @Inject() ( + utegCandidateSource: UtegCandidateSource) { + + def build[Query <: PipelineQuery with HasExcludedIds]( + identifierPrefix: String + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): UTEGCandidatePipelineConfig[Query] = { + new UTEGCandidatePipelineConfig( + identifierPrefix, + utegCandidateSource + ) + } +} + +class UTEGCandidatePipelineConfig[Query <: PipelineQuery with HasExcludedIds]( + identifierPrefix: String, + utegCandidateSource: UtegCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + uteg.RecommendTweetEntityRequest, + uteg.TweetRecommendation, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.UTEG) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "UTEGEnabled", + param = UTEGEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + uteg.RecommendTweetEntityRequest + ] = UtegQueryTransformer(identifier) + + override def candidateSource: CandidateSource[ + uteg.RecommendTweetEntityRequest, + uteg.TweetRecommendation, + ] = utegCandidateSource + + override val featuresFromCandidateSourceTransformers = Seq(UtegResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + uteg.TweetRecommendation, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGExpansionTweetBasedCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGExpansionTweetBasedCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..2872ca181 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGExpansionTweetBasedCandidatePipelineConfigFactory.scala @@ -0,0 +1,125 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.UTG.UTGTweetBasedRequest +import com.twitter.tweet_mixer.candidate_source.UTG.UserTweetGraphConsumerBasedCandidateSource +import com.twitter.tweet_mixer.functional_component.gate.MaxFollowersGate +import com.twitter.tweet_mixer.functional_component.hydrator.OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.UTGOutlierSignalsQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.UTGTweetBasedQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UTGTweetBasedExpansionEnabled +import com.twitter.tweet_mixer.param.UTGParams._ +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UTGExpansionTweetBasedCandidatePipelineConfigFactory @Inject() ( + userTweetGraphConsumerBasedCandidateSource: UserTweetGraphConsumerBasedCandidateSource, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory: OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + utgOutlierSignalsQueryFeatureHydrator: UTGOutlierSignalsQueryFeatureHydrator) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): UTGExpansionTweetBasedCandidatePipelineConfig[Query] = { + new UTGExpansionTweetBasedCandidatePipelineConfig( + identifierPrefix, + signalsFn = signalsFn, + userTweetGraphConsumerBasedCandidateSource, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + utgOutlierSignalsQueryFeatureHydrator + ) + } +} + +class UTGExpansionTweetBasedCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long], + userTweetGraphConsumerBasedCandidateSource: UserTweetGraphConsumerBasedCandidateSource, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory: OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + utgOutlierSignalsQueryFeatureHydrator: UTGOutlierSignalsQueryFeatureHydrator +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + UTGTweetBasedRequest, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.UTGExpansionTweetBased) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate(name = "UTGTweetBasedExpansionEnabled", param = UTGTweetBasedExpansionEnabled), + MaxFollowersGate + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + UTGTweetBasedRequest + ] = UTGTweetBasedQueryTransformer( + identifier = TransformerIdentifier("UTGExpansionTweetBased"), + query => signalFnSelector(query), + isExpansionQuery = true, + minScoreParam = ConsumersBasedMinScoreParam, + degreeExponent = TweetBasedDegreeExponentParam + ) + + private def signalFnSelector(query: PipelineQuery): Seq[Long] = { + signalsFn(query) + } + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq( + ParamGatedQueryFeatureHydrator( + EnableTweetEmbeddingBasedFilteringParam, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory.build( + signalsFn + ) + ) + ) + + override val queryFeatureHydrationPhase2: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq( + ParamGatedQueryFeatureHydrator( + EnableTweetEmbeddingBasedFilteringParam, + utgOutlierSignalsQueryFeatureHydrator + ) + ) + + override def candidateSource: CandidateSource[ + UTGTweetBasedRequest, + TweetMixerCandidate + ] = userTweetGraphConsumerBasedCandidateSource + + override val featuresFromCandidateSourceTransformers = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGProducerBasedCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGProducerBasedCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..d9940a498 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGProducerBasedCandidatePipelineConfigFactory.scala @@ -0,0 +1,94 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.UTG.UTGProducerBasedRequest +import com.twitter.tweet_mixer.candidate_source.UTG.UserTweetGraphProducerBasedCandidateSource +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.UTGProducerBasedQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UTGProducerBasedEnabled +import com.twitter.tweet_mixer.param.UTGParams._ +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UTGProducerBasedCandidatePipelineConfigFactory @Inject() ( + userTweetGraphProducerBasedCandidateSource: UserTweetGraphProducerBasedCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): UTGProducerBasedCandidatePipelineConfig[Query] = { + new UTGProducerBasedCandidatePipelineConfig( + identifierPrefix, + signalFn, + userTweetGraphProducerBasedCandidateSource + ) + } +} + +class UTGProducerBasedCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long], + userTweetGraphProducerBasedCandidateSource: UserTweetGraphProducerBasedCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + UTGProducerBasedRequest, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.UTGProducerBased) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "UTGProducerBasedEnabled", + param = UTGProducerBasedEnabled + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + UTGProducerBasedRequest + ] = UTGProducerBasedQueryTransformer( + identifier = TransformerIdentifier("UTGProducerBased"), + signalsFn = signalsFn, + minScoreParam = TweetBasedMinScoreParam + ) + + override def candidateSource: CandidateSource[ + UTGProducerBasedRequest, + TweetMixerCandidate + ] = userTweetGraphProducerBasedCandidateSource + + override val featuresFromCandidateSourceTransformers = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGTweetBasedCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGTweetBasedCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..70c5b330e --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UTGTweetBasedCandidatePipelineConfigFactory.scala @@ -0,0 +1,125 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.UTG.UTGTweetBasedRequest +import com.twitter.tweet_mixer.candidate_source.UTG.UserTweetGraphTweetBasedCandidateSource +import com.twitter.tweet_mixer.functional_component.gate.MaxFollowersGate +import com.twitter.tweet_mixer.functional_component.hydrator.OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.UTGOutlierSignalsQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.UTGTweetBasedQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UTGTweetBasedEnabled +import com.twitter.tweet_mixer.param.UTGParams._ +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UTGTweetBasedCandidatePipelineConfigFactory @Inject() ( + userTweetGraphTweetBasedCandidateSource: UserTweetGraphTweetBasedCandidateSource, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory: OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + utgOutlierSignalsQueryFeatureHydrator: UTGOutlierSignalsQueryFeatureHydrator) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): UTGTweetBasedCandidatePipelineConfig[Query] = { + new UTGTweetBasedCandidatePipelineConfig( + identifierPrefix, + signalsFn = signalsFn, + userTweetGraphTweetBasedCandidateSource, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + utgOutlierSignalsQueryFeatureHydrator + ) + } +} + +class UTGTweetBasedCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long], + userTweetGraphTweetBasedCandidateSource: UserTweetGraphTweetBasedCandidateSource, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory: OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + utgOutlierSignalsQueryFeatureHydrator: UTGOutlierSignalsQueryFeatureHydrator +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + UTGTweetBasedRequest, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.UTGTweetBased) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate(name = "UTGTweetBasedEnabled", param = UTGTweetBasedEnabled), + MaxFollowersGate + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + UTGTweetBasedRequest + ] = UTGTweetBasedQueryTransformer( + identifier = TransformerIdentifier("UTGTweetBased"), + query => signalFnSelector(query), + isExpansionQuery = false, + minScoreParam = TweetBasedMinScoreParam, + degreeExponent = TweetBasedDegreeExponentParam + ) + + private def signalFnSelector(query: PipelineQuery): Seq[Long] = { + signalsFn(query) + } + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq( + ParamGatedQueryFeatureHydrator( + EnableTweetEmbeddingBasedFilteringParam, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory.build( + signalsFn + ) + ) + ) + + override val queryFeatureHydrationPhase2: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq( + ParamGatedQueryFeatureHydrator( + EnableTweetEmbeddingBasedFilteringParam, + utgOutlierSignalsQueryFeatureHydrator + ) + ) + + override def candidateSource: CandidateSource[ + UTGTweetBasedRequest, + TweetMixerCandidate + ] = userTweetGraphTweetBasedCandidateSource + + override val featuresFromCandidateSourceTransformers = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UVGExpansionTweetBasedCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UVGExpansionTweetBasedCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..da484cd4d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UVGExpansionTweetBasedCandidatePipelineConfigFactory.scala @@ -0,0 +1,123 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.UVG.UVGTweetBasedRequest +import com.twitter.tweet_mixer.candidate_source.UVG.UserVideoGraphConsumerBasedCandidateSource +import com.twitter.tweet_mixer.functional_component.gate.MaxFollowersGate +import com.twitter.tweet_mixer.functional_component.hydrator.OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.UVGOutlierSignalsQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.UVGTweetBasedQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UVGTweetBasedExpansionEnabled +import com.twitter.tweet_mixer.param.UVGParams._ +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UVGExpansionTweetBasedCandidatePipelineConfigFactory @Inject() ( + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory: OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + uvgOutlierSignalsQueryFeatureHydrator: UVGOutlierSignalsQueryFeatureHydrator, + userVideoGraphConsumerBasedCandidateSource: UserVideoGraphConsumerBasedCandidateSource) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): UVGExpansionTweetBasedCandidatePipelineConfig[Query] = { + new UVGExpansionTweetBasedCandidatePipelineConfig( + identifierPrefix, + signalFn, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + uvgOutlierSignalsQueryFeatureHydrator, + userVideoGraphConsumerBasedCandidateSource + ) + } +} + +class UVGExpansionTweetBasedCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long], + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory: OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + uvgOutlierSignalsQueryFeatureHydrator: UVGOutlierSignalsQueryFeatureHydrator, + userVideoGraphConsumerBasedCandidateSource: UserVideoGraphConsumerBasedCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + UVGTweetBasedRequest, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.UVGExpansionTweetBased) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate(name = "UVGTweetBasedExpansionEnabled", param = UVGTweetBasedExpansionEnabled), + MaxFollowersGate + ) + + private def signalFnSelector(query: PipelineQuery): Seq[Long] = { + signalsFn(query) + } + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + UVGTweetBasedRequest + ] = UVGTweetBasedQueryTransformer( + identifier = TransformerIdentifier("UVGExpansion"), + signalsFn = signalFnSelector, + isExpansionQuery = true, + minScoreParam = TweetBasedMinScoreParam, + degreeExponent = TweetBasedDegreeExponentParam + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq( + ParamGatedQueryFeatureHydrator( + EnableTweetEmbeddingBasedFilteringParam, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory.build( + signalsFn + ) + ) + ) + + override val queryFeatureHydrationPhase2: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq( + ParamGatedQueryFeatureHydrator( + EnableTweetEmbeddingBasedFilteringParam, + uvgOutlierSignalsQueryFeatureHydrator + ) + ) + + override def candidateSource: CandidateSource[ + UVGTweetBasedRequest, + TweetMixerCandidate + ] = userVideoGraphConsumerBasedCandidateSource + + override val featuresFromCandidateSourceTransformers = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts = Seq( + defaultSuccessRateAlert(), + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UVGTweetBasedCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UVGTweetBasedCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..e452c3cd0 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UVGTweetBasedCandidatePipelineConfigFactory.scala @@ -0,0 +1,126 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.UVG.UVGTweetBasedRequest +import com.twitter.tweet_mixer.candidate_source.UVG.UserVideoGraphTweetBasedCandidateSource +import com.twitter.tweet_mixer.functional_component.gate.MaxFollowersGate +import com.twitter.tweet_mixer.functional_component.hydrator.OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.UVGOutlierSignalsQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.functional_component.transformer.UVGTweetBasedQueryTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UVGTweetBasedEnabled +import com.twitter.tweet_mixer.param.UVGParams._ +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UVGTweetBasedCandidatePipelineConfigFactory @Inject() ( + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory: OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + uvgOutlierSignalsQueryFeatureHydrator: UVGOutlierSignalsQueryFeatureHydrator, + userVideoGraphTweetBasedCandidateSource: UserVideoGraphTweetBasedCandidateSource, +) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long] + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): UVGTweetBasedCandidatePipelineConfig[Query] = { + new UVGTweetBasedCandidatePipelineConfig( + identifierPrefix, + signalsFn = signalsFn, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + uvgOutlierSignalsQueryFeatureHydrator, + userVideoGraphTweetBasedCandidateSource + ) + } +} + +class UVGTweetBasedCandidatePipelineConfig[Query <: PipelineQuery]( + identifierPrefix: String, + signalsFn: PipelineQuery => Seq[Long], + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory: OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory, + uvgOutlierSignalsQueryFeatureHydrator: UVGOutlierSignalsQueryFeatureHydrator, + userVideoGraphTweetBasedCandidateSource: UserVideoGraphTweetBasedCandidateSource +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + UVGTweetBasedRequest, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.UVGTweetBased) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate(name = "UVGTweetBasedEnabled", param = UVGTweetBasedEnabled), + MaxFollowersGate + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + UVGTweetBasedRequest + ] = UVGTweetBasedQueryTransformer( + identifier = TransformerIdentifier("UVG"), + query => signalFnSelector(query), + isExpansionQuery = false, + minScoreParam = TweetBasedMinScoreParam, + degreeExponent = TweetBasedDegreeExponentParam + ) + + private def signalFnSelector(query: PipelineQuery): Seq[Long] = { + signalsFn(query) + } + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq( + ParamGatedQueryFeatureHydrator( + EnableTweetEmbeddingBasedFilteringParam, + outlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory.build( + signalsFn + ) + ) + ) + + override val queryFeatureHydrationPhase2: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq( + ParamGatedQueryFeatureHydrator( + EnableTweetEmbeddingBasedFilteringParam, + uvgOutlierSignalsQueryFeatureHydrator + ) + ) + + override def candidateSource: CandidateSource[ + UVGTweetBasedRequest, + TweetMixerCandidate + ] = userVideoGraphTweetBasedCandidateSource + + override val featuresFromCandidateSourceTransformers = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UserInterestsSummaryCandidatePipelineConfigFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UserInterestsSummaryCandidatePipelineConfigFactory.scala new file mode 100644 index 000000000..020df461b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UserInterestsSummaryCandidatePipelineConfigFactory.scala @@ -0,0 +1,151 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.gate.ParamGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.tweet_mixer.candidate_source.ndr_ann.ContentEmbeddingUserANNKey +import com.twitter.tweet_mixer.candidate_source.ndr_ann.UserInterestEmbeddingANNCandidateSource +import com.twitter.tweet_mixer.candidate_source.ndr_ann.UserInterestEmbeddingANNCandidateSourceFactory +import com.twitter.tweet_mixer.candidate_source.ndr_ann.ContentEmbeddingMultipleUserANNQuery +import com.twitter.tweet_mixer.functional_component.gate.MinTimeSinceLastRequestGate +import com.twitter.tweet_mixer.functional_component.hydrator.UserInterestSummaryEmbeddingFeature +import com.twitter.tweet_mixer.functional_component.hydrator.UserInterestSummaryEmbeddingQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UserInterestSummarySimilarityMaxCandidates +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UserInterestSummarySimilarityScoreThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UserInterestSummarySimilarityEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UserInterestSummarySimilarityVectorDBCollectionName +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserInterestsSummaryCandidatePipelineConfigFactory @Inject() ( + contentEmbeddingANNCandidateSourceFactory: UserInterestEmbeddingANNCandidateSourceFactory, + userInterestSummaryEmbeddingQueryFeatureHydratorFactory: UserInterestSummaryEmbeddingQueryFeatureHydratorFactory, + statsReceiver: StatsReceiver) { + + def build[Query <: PipelineQuery]( + identifierPrefix: String + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): UserInterestsSummaryCandidatePipelineConfig[Query] = { + val candidateSource = contentEmbeddingANNCandidateSourceFactory.build( + CandidatePipelineConstants.UserInterestSummary + ) + new UserInterestsSummaryCandidatePipelineConfig( + candidateSource, + userInterestSummaryEmbeddingQueryFeatureHydratorFactory, + identifierPrefix, + statsReceiver + ) + } +} + +class UserInterestsSummaryCandidatePipelineConfig[Query <: PipelineQuery]( + contentEmbeddingANNCandidateSource: UserInterestEmbeddingANNCandidateSource, + userInterestSummaryEmbeddingQueryFeatureHydratorFactory: UserInterestSummaryEmbeddingQueryFeatureHydratorFactory, + identifierPrefix: String, + statsReceiver: StatsReceiver +)( + implicit notificationGroup: Map[String, NotificationGroup]) + extends CandidatePipelineConfig[ + Query, + ContentEmbeddingMultipleUserANNQuery, + TweetMixerCandidate, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.UserInterestSummary + ) + + override val gates: Seq[Gate[PipelineQuery]] = Seq( + ParamGate( + name = "UserInterestsSummary", + param = UserInterestSummarySimilarityEnabled + ), + MinTimeSinceLastRequestGate + ) + + override val queryFeatureHydration: Seq[BaseQueryFeatureHydrator[PipelineQuery, _]] = Seq( + userInterestSummaryEmbeddingQueryFeatureHydratorFactory.build() + ) + + private val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetIdsLenStats = scopedStats.stat("tweetIdsLen") + private val embeddingLenStats = scopedStats.stat("embeddingLen") + + override val queryTransformer: CandidatePipelineQueryTransformer[ + Query, + ContentEmbeddingMultipleUserANNQuery + ] = { query => + val userId = query.getRequiredUserId + val collectionName = query.params(UserInterestSummarySimilarityVectorDBCollectionName) + val maxCandidates = query.params(UserInterestSummarySimilarityMaxCandidates) + val embeddings = query.features + .flatMap(_.getOrElse(UserInterestSummaryEmbeddingFeature, None)) + val scoreThreshold = query.params(UserInterestSummarySimilarityScoreThreshold) + + val tweetSize: Int = 0 // Placeholder, as we're not using tweet IDs directly + val embeddingSize: Int = embeddings.map(_.size).getOrElse(0) + tweetIdsLenStats.add(tweetSize) + embeddingLenStats.add(embeddingSize) + + val tweetsANNKeys: Seq[ContentEmbeddingUserANNKey] = embeddings match { + case Some(embeddingSeqs) if embeddingSeqs.nonEmpty => + embeddingSeqs.zipWithIndex.map { + case (embeddingSeq, index) => + ContentEmbeddingUserANNKey( + index.toLong, // Using index as a placeholder since we don't have tweet IDs + userId, + Some(embeddingSeq), + collectionName, + maxCandidates, + scoreThreshold + ) + } + case _ => + Seq.empty[ContentEmbeddingUserANNKey] + } + + ContentEmbeddingMultipleUserANNQuery( + annKeys = tweetsANNKeys, + enableCache = true + ) + } + + override def candidateSource: CandidateSource[ + ContentEmbeddingMultipleUserANNQuery, + TweetMixerCandidate + ] = contentEmbeddingANNCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => + TweetCandidate(id = candidate.tweetId) + } + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UserLocationCandidatePipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UserLocationCandidatePipelineConfig.scala new file mode 100644 index 000000000..c517c10b4 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline/UserLocationCandidatePipelineConfig.scala @@ -0,0 +1,79 @@ +package com.twitter.tweet_mixer.candidate_pipeline + +import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature +import com.twitter.product_mixer.component_library.gate.DefinedLocationGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.strato.generated.client.timelines.local.FetchLocalPostsClientColumn +import com.twitter.tweet_mixer.candidate_source.user_location.UserLocationCandidateSource +import com.twitter.tweet_mixer.functional_component.gate.AllowLowSignalUserGate +import com.twitter.tweet_mixer.functional_component.transformer.TweetMixerCandidateFeatureTransformer +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserLocationCandidatePipelineConfig @Inject() ( + userLocationCandidateSource: UserLocationCandidateSource) + extends CandidatePipelineConfig[ + PipelineQuery, + StratoKeyView[ + FetchLocalPostsClientColumn.Key, + Unit + ], + TweetMixerCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier(CandidatePipelineConstants.UserLocation) + + override val gates: Seq[Gate[PipelineQuery]] = Seq(DefinedLocationGate, AllowLowSignalUserGate) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + StratoKeyView[ + FetchLocalPostsClientColumn.Key, + Unit + ] + ] = { query => + val location = query.features.flatMap(_.get(LocationFeature)).head + + val locationString = Seq(location.city, location.metro, location.region) + .map { place => place.map { l => l.toString }.getOrElse("") }.mkString(",") + + StratoKeyView(key = locationString, Unit) + } + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetMixerCandidate, + TweetCandidate + ] = { candidate => TweetCandidate(id = candidate.tweetId) } + + override def candidateSource: CandidateSource[ + StratoKeyView[FetchLocalPostsClientColumn.Key, Unit], + TweetMixerCandidate, + ] = userLocationCandidateSource + + override def featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetMixerCandidate] + ] = Seq(TweetMixerCandidateFeatureTransformer) + + override val alerts: Seq[Alert] = Seq( + defaultSuccessRateAlert()(ForYouGroupMap), + defaultEmptyResponseRateAlert()(ForYouGroupMap) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/BUILD.bazel new file mode 100644 index 000000000..54e356a77 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/BUILD.bazel @@ -0,0 +1,8 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/BUILD.bazel new file mode 100644 index 000000000..b03402ffb --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/BUILD.bazel @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "servo/repo", + "src/thrift/com/twitter/recos/user_tweet_graph:user_tweet_graph-scala", + "strato/config/columns/recommendations/twistly:twistly-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/engaged_users", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UTGProducerBasedRequest.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UTGProducerBasedRequest.scala new file mode 100644 index 000000000..9184382fa --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UTGProducerBasedRequest.scala @@ -0,0 +1,15 @@ +package com.twitter.tweet_mixer.candidate_source.UTG + +import com.twitter.recos.user_tweet_graph.thriftscala.RelatedTweetSimilarityAlgorithm +import com.twitter.tweet_mixer.feature.EntityTypes.UserId + +case class UTGProducerBasedRequest( + seedUserIds: Seq[UserId], + maxResults: Option[Int], + minCooccurrence: Option[Int], + minScore: Option[Double], + maxNumFollowers: Option[Int], + maxTweetAgeInHours: Option[Int], + similarityAlgorithm: Option[RelatedTweetSimilarityAlgorithm] = + Some(RelatedTweetSimilarityAlgorithm.LogCosine), + enableCache: Boolean = true) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UTGTweetBasedRequest.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UTGTweetBasedRequest.scala new file mode 100644 index 000000000..91933ac9a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UTGTweetBasedRequest.scala @@ -0,0 +1,16 @@ +package com.twitter.tweet_mixer.candidate_source.UTG + +import com.twitter.recos.user_tweet_graph.thriftscala.RelatedTweetSimilarityAlgorithm +import com.twitter.tweet_mixer.feature.EntityTypes.TweetId + +case class UTGTweetBasedRequest( + seedTweetIds: Seq[TweetId], + maxResults: Option[Int], + minCooccurrence: Option[Int], + minScore: Option[Double], + maxTweetAgeInHours: Option[Int], + maxConsumerSeeds: Option[Int] = None, + similarityAlgorithm: Option[RelatedTweetSimilarityAlgorithm] = + Some(RelatedTweetSimilarityAlgorithm.LogCosine), + enableCache: Boolean = true, + degreeExponent: Option[Double] = None) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphConsumerBasedCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphConsumerBasedCandidateSource.scala new file mode 100644 index 000000000..00368f07b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphConsumerBasedCandidateSource.scala @@ -0,0 +1,114 @@ +package com.twitter.tweet_mixer.candidate_source.UTG + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.finagle.util.DefaultTimer +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.recos.user_tweet_graph.thriftscala.ConsumersBasedRelatedTweetRequest +import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.candidate_source.engaged_users.RecentEngagedUsersCandidateSource +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserTweetGraphConsumerBasedCandidateSource @Inject() ( + utgClient: UserTweetGraph.MethodPerEndpoint, + recentEngagedUsersCandidateSource: RecentEngagedUsersCandidateSource, + memcacheClient: MemcacheStitchClient) + extends MemcachedCandidateSource[ + UTGTweetBasedRequest, + (ConsumersBasedRelatedTweetRequest, Long), + (Long, Double), + TweetMixerCandidate + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "UserTweetGraphConsumerBasedCandidateSource" + ) + + private val defaultVersion: Long = 0 + + implicit val timer = DefaultTimer + + override val TTL = Utils.randomizedTTL(600) //10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: (ConsumersBasedRelatedTweetRequest, Long)): String = + key match { + case (request, seedId) => + "UTGExpansion:" + + seedId.toString + + request.minScore.toString + + request.maxResults.toString + + request.maxTweetAgeInHours.toString + + request.minCooccurrence.toString + + request.similarityAlgorithm.toString + } + + override val valueTransformer: Transformer[Seq[(Long, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + override def enableCache(request: UTGTweetBasedRequest): Boolean = request.enableCache + + override def getKeys( + request: UTGTweetBasedRequest + ): Stitch[Seq[(ConsumersBasedRelatedTweetRequest, Long)]] = { + Stitch.traverse(request.seedTweetIds) { seedId => + val recentEngagedUsersStitch = + recentEngagedUsersCandidateSource + .apply((seedId, defaultVersion)) + .within(50.millis) + .handle { case _ => Seq.empty } + val consumerBasedRequestStitch = recentEngagedUsersStitch.map { recentEngagedUsers => + ConsumersBasedRelatedTweetRequest( + recentEngagedUsers.take(request.maxConsumerSeeds.getOrElse(0)), + maxResults = request.maxResults, + minCooccurrence = request.minCooccurrence, + excludeTweetIds = Some(Seq(seedId)), + minScore = request.minScore, + maxTweetAgeInHours = request.maxTweetAgeInHours, + similarityAlgorithm = request.similarityAlgorithm + ) + } + consumerBasedRequestStitch.map { + (_, seedId) + } + } + } + + override def getCandidatesFromStore( + key: (ConsumersBasedRelatedTweetRequest, Long) + ): Stitch[Seq[(Long, Double)]] = { + Stitch + .callFuture(utgClient.consumersBasedRelatedTweets(key._1)) + .map { response => + response.tweets.map { relatedTweet => + (relatedTweet.tweetId, relatedTweet.score) + } + } + } + + override def postProcess( + request: UTGTweetBasedRequest, + keys: Seq[(ConsumersBasedRelatedTweetRequest, Long)], + resultsSeq: Seq[Seq[(Long, Double)]] + ): Seq[TweetMixerCandidate] = { + val utgCandidates = keys.zip(resultsSeq).map { + case (key, results) => + val seedId = key._2 + results + .map { + case (id, score) => TweetMixerCandidate(id, score, seedId) + } + } + TweetMixerCandidate.interleave(utgCandidates) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphProducerBasedCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphProducerBasedCandidateSource.scala new file mode 100644 index 000000000..7666c304d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphProducerBasedCandidateSource.scala @@ -0,0 +1,97 @@ +package com.twitter.tweet_mixer.candidate_source.UTG + +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.recos.user_tweet_graph.thriftscala.ProducerBasedRelatedTweetRequest +import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserTweetGraphProducerBasedCandidateSource @Inject() ( + utgClient: UserTweetGraph.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient) + extends MemcachedCandidateSource[ + UTGProducerBasedRequest, + ProducerBasedRelatedTweetRequest, + (Long, Double), + TweetMixerCandidate + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "UserTweetGraphProducerBasedCandidateSource" + ) + + override val TTL = Utils.randomizedTTL(600) //10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: ProducerBasedRelatedTweetRequest): String = { + "UTG:" + + key.producerId.toString + + key.minScore.toString + + key.maxResults.toString + + key.maxNumFollowers.toString + + key.maxTweetAgeInHours.toString + + key.minCooccurrence.toString + + key.similarityAlgorithm.toString + } + + val valueTransformer: Transformer[Seq[(Long, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + override def enableCache(request: UTGProducerBasedRequest): Boolean = request.enableCache + + override def getKeys( + request: UTGProducerBasedRequest + ): Stitch[Seq[ProducerBasedRelatedTweetRequest]] = { + Stitch.value { + request.seedUserIds.map { seedUserId => + ProducerBasedRelatedTweetRequest( + seedUserId, + maxResults = request.maxResults, + minCooccurrence = request.minCooccurrence, + minScore = request.minScore, + maxNumFollowers = request.maxNumFollowers, + maxTweetAgeInHours = request.maxTweetAgeInHours, + similarityAlgorithm = request.similarityAlgorithm + ) + } + } + } + + override def getCandidatesFromStore( + key: ProducerBasedRelatedTweetRequest + ): Stitch[Seq[(Long, Double)]] = { + Stitch + .callFuture(utgClient.producerBasedRelatedTweets(key)) + .map { response => + response.tweets.map { relatedTweet => + (relatedTweet.tweetId, relatedTweet.score) + } + } + } + + override def postProcess( + request: UTGProducerBasedRequest, + keys: Seq[ProducerBasedRelatedTweetRequest], + resultsSeq: Seq[Seq[(Long, Double)]] + ): Seq[TweetMixerCandidate] = { + val utgCandidates = keys.zip(resultsSeq).map { + case (key, results) => + val seedId = key.producerId + results + .map { + case (id, score) => TweetMixerCandidate(id, score, seedId) + } + } + TweetMixerCandidate.interleave(utgCandidates) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphTweetBasedCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphTweetBasedCandidateSource.scala new file mode 100644 index 000000000..b376760bb --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG/UserTweetGraphTweetBasedCandidateSource.scala @@ -0,0 +1,98 @@ +package com.twitter.tweet_mixer.candidate_source.UTG + +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.recos.user_tweet_graph.thriftscala.TweetBasedRelatedTweetRequest +import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserTweetGraphTweetBasedCandidateSource @Inject() ( + utgClient: UserTweetGraph.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient) + extends MemcachedCandidateSource[ + UTGTweetBasedRequest, + TweetBasedRelatedTweetRequest, + (Long, Double), + TweetMixerCandidate + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "UserTweetGraphTweetBasedCandidateSource" + ) + + override val TTL = Utils.randomizedTTL(600) //10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: TweetBasedRelatedTweetRequest): String = { + "UTG:" + + key.tweetId.toString + + key.minScore.toString + + key.maxResults.toString + + key.maxTweetAgeInHours.toString + + key.minCooccurrence.toString + + key.similarityAlgorithm.toString + + key.degreeExponent + } + + val valueTransformer: Transformer[Seq[(Long, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + override def enableCache(request: UTGTweetBasedRequest): Boolean = request.enableCache + + override def getKeys( + request: UTGTweetBasedRequest + ): Stitch[Seq[TweetBasedRelatedTweetRequest]] = { + Stitch.value { + request.seedTweetIds.map { seedTweetId => + TweetBasedRelatedTweetRequest( + seedTweetId, + maxResults = request.maxResults, + minCooccurrence = request.minCooccurrence, + excludeTweetIds = Some(Seq(seedTweetId)), + minScore = request.minScore, + maxTweetAgeInHours = request.maxTweetAgeInHours, + similarityAlgorithm = request.similarityAlgorithm, + degreeExponent = request.degreeExponent + ) + } + } + } + + override def getCandidatesFromStore( + key: TweetBasedRelatedTweetRequest + ): Stitch[Seq[(Long, Double)]] = { + Stitch + .callFuture(utgClient.tweetBasedRelatedTweets(key)) + .map { response => + response.tweets.map { relatedTweet => + (relatedTweet.tweetId, relatedTweet.score) + } + } + } + + override def postProcess( + request: UTGTweetBasedRequest, + keys: Seq[TweetBasedRelatedTweetRequest], + resultsSeq: Seq[Seq[(Long, Double)]] + ): Seq[TweetMixerCandidate] = { + val utgCandidates = keys.zip(resultsSeq).map { + case (key, results) => + val seedId = key.tweetId + results + .map { + case (id, score) => TweetMixerCandidate(id, score, seedId) + } + } + TweetMixerCandidate.interleave(utgCandidates) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/BUILD.bazel new file mode 100644 index 000000000..ca4ffb15d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/BUILD.bazel @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "servo/repo", + "src/thrift/com/twitter/recos/user_video_graph:user_video_graph-scala", + "strato/config/columns/recommendations/twistly:twistly-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/engaged_users", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UVGTweetBasedRequest.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UVGTweetBasedRequest.scala new file mode 100644 index 000000000..bfa83c571 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UVGTweetBasedRequest.scala @@ -0,0 +1,20 @@ +package com.twitter.tweet_mixer.candidate_source.UVG + +import com.twitter.recos.user_video_graph.thriftscala.RelatedTweetSimilarityAlgorithm +import com.twitter.tweet_mixer.feature.EntityTypes.TweetId + +case class UVGTweetBasedRequest( + seedTweetIds: Seq[TweetId], + maxResults: Option[Int], + minCooccurrence: Option[Int], + minScore: Option[Double], + maxTweetAgeInHours: Option[Int], + maxConsumerSeeds: Option[Int] = None, + similarityAlgorithm: Option[RelatedTweetSimilarityAlgorithm] = + Some(RelatedTweetSimilarityAlgorithm.LogCosine), + enableCache: Boolean = true, + maxNumSamplesPerNeighbor: Option[Int] = None, + maxLeftNodeDegree: Option[Int] = None, + maxRightNodeDegree: Option[Int] = None, + sampleRHSTweets: Option[Boolean] = None, + degreeExponent: Option[Double] = None) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UserVideoGraphConsumerBasedCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UserVideoGraphConsumerBasedCandidateSource.scala new file mode 100644 index 000000000..393a8c7ff --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UserVideoGraphConsumerBasedCandidateSource.scala @@ -0,0 +1,115 @@ +package com.twitter.tweet_mixer.candidate_source.UVG + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.finagle.util.DefaultTimer +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.recos.user_video_graph.thriftscala.ConsumersBasedRelatedTweetRequest +import com.twitter.recos.user_video_graph.thriftscala.UserVideoGraph +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.candidate_source.engaged_users.RecentEngagedUsersCandidateSource +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserVideoGraphConsumerBasedCandidateSource @Inject() ( + uvgClient: UserVideoGraph.MethodPerEndpoint, + recentEngagedUsersCandidateSource: RecentEngagedUsersCandidateSource, + memcacheClient: MemcacheStitchClient) + extends MemcachedCandidateSource[ + UVGTweetBasedRequest, + (ConsumersBasedRelatedTweetRequest, Long), + (Long, Double), + TweetMixerCandidate + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "UserVideoGraphConsumerBasedCandidateSource" + ) + + private val defaultVersion: Long = 0 + + implicit val timer = DefaultTimer + + override val TTL = Utils.randomizedTTL(600) //10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: (ConsumersBasedRelatedTweetRequest, Long)): String = { + key match { + case (request, seedId) => + "UVGExpansion:" + + seedId.toString + + request.minScore.toString + + request.maxResults.toString + + request.maxTweetAgeInHours.toString + + request.minCooccurrence.toString + + request.similarityAlgorithm.toString + } + } + + val valueTransformer: Transformer[Seq[(Long, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + override def enableCache(request: UVGTweetBasedRequest): Boolean = request.enableCache + + override def getKeys( + request: UVGTweetBasedRequest + ): Stitch[Seq[(ConsumersBasedRelatedTweetRequest, Long)]] = { + Stitch.traverse(request.seedTweetIds) { seedId => + val recentEngagedUsersStitch = + recentEngagedUsersCandidateSource + .apply((seedId, defaultVersion)) + .within(50.millis) + .handle { case _ => Seq.empty } + val consumerBasedRequestStitch = recentEngagedUsersStitch.map { recentEngagedUsers => + ConsumersBasedRelatedTweetRequest( + recentEngagedUsers.take(request.maxConsumerSeeds.getOrElse(0)), + maxResults = request.maxResults, + minCooccurrence = request.minCooccurrence, + excludeTweetIds = Some(Seq(seedId)), + minScore = request.minScore, + maxTweetAgeInHours = request.maxTweetAgeInHours, + similarityAlgorithm = request.similarityAlgorithm + ) + } + consumerBasedRequestStitch.map { + (_, seedId) + } + } + } + + override def getCandidatesFromStore( + key: (ConsumersBasedRelatedTweetRequest, Long) + ): Stitch[Seq[(Long, Double)]] = { + Stitch + .callFuture(uvgClient.consumersBasedRelatedTweets(key._1)) + .map { response => + response.tweets.map { relatedTweet => + (relatedTweet.tweetId, relatedTweet.score) + } + } + } + + override def postProcess( + request: UVGTweetBasedRequest, + keys: Seq[(ConsumersBasedRelatedTweetRequest, Long)], + resultsSeq: Seq[Seq[(Long, Double)]] + ): Seq[TweetMixerCandidate] = { + val uvgCandidates = keys.zip(resultsSeq).map { + case (key, results) => + val seedId = key._2 + results + .map { + case (id, score) => TweetMixerCandidate(id, score, seedId) + } + } + TweetMixerCandidate.interleave(uvgCandidates) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UserVideoGraphTweetBasedCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UserVideoGraphTweetBasedCandidateSource.scala new file mode 100644 index 000000000..286186ced --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG/UserVideoGraphTweetBasedCandidateSource.scala @@ -0,0 +1,106 @@ +package com.twitter.tweet_mixer.candidate_source.UVG + +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.recos.user_video_graph.thriftscala.TweetBasedRelatedTweetRequest +import com.twitter.recos.user_video_graph.thriftscala.UserVideoGraph +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserVideoGraphTweetBasedCandidateSource @Inject() ( + uvgClient: UserVideoGraph.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient) + extends MemcachedCandidateSource[ + UVGTweetBasedRequest, + TweetBasedRelatedTweetRequest, + (Long, Double), + TweetMixerCandidate + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "UserVideoGraphTweetBasedCandidateSource" + ) + + override val TTL = Utils.randomizedTTL(600) //10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: TweetBasedRelatedTweetRequest): String = { + "UVG:" + + key.tweetId.toString + + key.minScore.toString + + key.maxResults.toString + + key.maxTweetAgeInHours.toString + + key.minCooccurrence.toString + + key.similarityAlgorithm.toString + + key.maxNumSamplesPerNeighbor + + key.maxRightNodeDegree + + key.maxLeftNodeDegree + + key.sampleRHSTweets + + key.degreeExponent + } + + val valueTransformer: Transformer[Seq[(Long, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + override def enableCache(request: UVGTweetBasedRequest): Boolean = request.enableCache + + override def getKeys( + request: UVGTweetBasedRequest + ): Stitch[Seq[TweetBasedRelatedTweetRequest]] = { + Stitch.value { + request.seedTweetIds.map { seedTweetId => + TweetBasedRelatedTweetRequest( + seedTweetId, + maxResults = request.maxResults, + minCooccurrence = request.minCooccurrence, + excludeTweetIds = Some(Seq(seedTweetId)), + minScore = request.minScore, + maxTweetAgeInHours = request.maxTweetAgeInHours, + similarityAlgorithm = request.similarityAlgorithm, + maxNumSamplesPerNeighbor = request.maxNumSamplesPerNeighbor, + maxLeftNodeDegree = request.maxLeftNodeDegree, + maxRightNodeDegree = request.maxRightNodeDegree, + sampleRHSTweets = request.sampleRHSTweets, + degreeExponent = request.degreeExponent + ) + } + } + } + + override def getCandidatesFromStore( + key: TweetBasedRelatedTweetRequest + ): Stitch[Seq[(Long, Double)]] = { + Stitch + .callFuture(uvgClient.tweetBasedRelatedTweets(key)) + .map { response => + response.tweets.map { relatedTweet => + (relatedTweet.tweetId, relatedTweet.score) + } + } + } + + override def postProcess( + request: UVGTweetBasedRequest, + keys: Seq[TweetBasedRelatedTweetRequest], + resultsSeq: Seq[Seq[(Long, Double)]] + ): Seq[TweetMixerCandidate] = { + val uvgCandidates = keys.zip(resultsSeq).map { + case (key, results) => + val seedId = key.tweetId + results + .map { + case (id, score) => TweetMixerCandidate(id, score, seedId) + } + } + TweetMixerCandidate.interleave(uvgCandidates) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source/BUILD.bazel new file mode 100644 index 000000000..126fbd05b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source/BUILD.bazel @@ -0,0 +1,12 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "servo/repo", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source/MemcachedCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source/MemcachedCandidateSource.scala new file mode 100644 index 000000000..3dae9992b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source/MemcachedCandidateSource.scala @@ -0,0 +1,90 @@ +package com.twitter.tweet_mixer.candidate_source.cached_candidate_source + +import com.twitter.io.Buf +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.util.Return + +trait MemcachedCandidateSource[Request, Key, Value, Response] + extends CandidateSource[Request, Response] { + + val memcache: MemcacheStitchClient + + val TTL: Int // memcache TTL + + def keyTransformer(key: Key): String + + val valueTransformer: Transformer[Seq[Value], Buf] + + def enableCache(request: Request): Boolean = true + + def getKeys(request: Request): Stitch[Seq[Key]] // get keys from request + + def getCandidatesFromStore( + key: Key + ): Stitch[Seq[Value]] // get set of candidates from underlying store + + def postProcess( + request: Request, + keys: Seq[Key], + results: Seq[Seq[Value]] + ): Seq[Response] // postProcess and convert to Response type + + override def apply(request: Request): Stitch[Seq[Response]] = { + OffloadFuturePools.offloadStitch { + getKeys(request).flatMap { keys => + if (keys.isEmpty) Stitch.value(Seq.empty) + else { + val resultsStitch = if (enableCache(request)) { + Stitch + .collectToTry(keys.map(getCandidates)).map(_.map { + case Return(result) => result + case _ => Seq.empty[Value] + }) + } else { + Stitch + .collectToTry(keys.map(getAndSetCandidates)).map(_.map { + case Return(result) => result + case _ => Seq.empty[Value] + }) + } + resultsStitch + .map(postProcess(request, keys, _)) + .handle { case _ => Seq.empty } + } + } + } + } + + private def getCandidates( + key: Key + ): Stitch[Seq[Value]] = { + memcache + .get(keyTransformer(key)) + .flatMap { + case Some(value) => + Stitch + .value( + valueTransformer + .from(value) + .getOrElse(Seq.empty) + ).rescue { + case _ => getAndSetCandidates(key) + } + case None => getAndSetCandidates(key) + } + } + + private def getAndSetCandidates( + key: Key + ): Stitch[Seq[Value]] = { + val memcacheKey: String = keyTransformer(key) + val memcacheSetFunction: Seq[Value] => Stitch[Unit] = { result => + Stitch.async(memcache.set(memcacheKey, valueTransformer.to(result).getOrElse(Buf.Empty), TTL)) + } + getCandidatesFromStore(key).applyEffect(memcacheSetFunction) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/content_embedding_ann/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/content_embedding_ann/BUILD.bazel new file mode 100644 index 000000000..f294b0b97 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/content_embedding_ann/BUILD.bazel @@ -0,0 +1,16 @@ +scala_library( + compiler_option_sets = ["fatal_warnings"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "servo/repo/src/main/scala", + "strato/config/columns/searchai/candidatesources:candidatesources-strato-client", + "strato/config/src/thrift/com/twitter/strato/columns/searchai:searchai-scala", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + ], + sources = ["*.scala"], + strict_deps = True, + tags = ["bazel-compatible"], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/content_embedding_ann/ContentEmbeddingAnnCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/content_embedding_ann/ContentEmbeddingAnnCandidateSource.scala new file mode 100644 index 000000000..37be11b53 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/content_embedding_ann/ContentEmbeddingAnnCandidateSource.scala @@ -0,0 +1,88 @@ +package com.twitter.tweet_mixer.candidate_source.content_embedding_ann + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ExpiringLruInProcessCache +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Client +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.searchai.candidatesources.RealtimeTopCandidatesBlendedOnTweetClientColumn +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.util.Random + +object ContentEmbeddingAnnCandidateSource { + private val BaseTTL = 5 + private val TTL = (BaseTTL + Random.nextInt(5)).minutes + + val cache: InProcessCache[ContentEmbeddingAnnQuery, Seq[TweetMixerCandidate]] = + new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 500) +} + +case class ContentEmbeddingAnnQuery( + seedPostId: Seq[Long], + maxCandidates: Int, + minScore: Double, + maxScore: Double, + countryCode: String, + languageCode: String, + decayByCountry: Boolean, + includeMediaSource: Boolean, + includeTextSource: Boolean) + +@Singleton +class ContentEmbeddingAnnCandidateSource @Inject() ( + @Named("StratoClientWithModerateTimeout") stratoClient: Client) + extends CandidateSource[ + ContentEmbeddingAnnQuery, + TweetMixerCandidate + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "ContentEmbeddingAnn") + + private val fetcher: Fetcher[ + RealtimeTopCandidatesBlendedOnTweetClientColumn.Key, + RealtimeTopCandidatesBlendedOnTweetClientColumn.View, + RealtimeTopCandidatesBlendedOnTweetClientColumn.Value + ] = new RealtimeTopCandidatesBlendedOnTweetClientColumn(stratoClient).fetcher + + override def apply( + key: ContentEmbeddingAnnQuery + ): Stitch[Seq[TweetMixerCandidate]] = OffloadFuturePools.offloadStitch { + Stitch + .traverse(key.seedPostId) { id => + fetcher + .fetch( + id, + RealtimeTopCandidatesBlendedOnTweetClientColumn.View( + limit = key.maxCandidates, + minScoreThreshold = key.minScore, + maxSimilarity = key.maxScore, + countryCode = key.countryCode, + languageCode = key.languageCode, + decayByCountry = key.decayByCountry, + includeMediaSource = key.includeMediaSource, + includeTextSource = key.includeTextSource + ) + ).map { result => + val candidates = result.v.getOrElse(Seq.empty).map { post => + TweetMixerCandidate( + tweetId = post.tweetId, + score = post.score, + seedId = id + ) + } + ContentEmbeddingAnnCandidateSource.cache.set(key, candidates) + candidates + }.liftToOption().map { v => + v.getOrElse(Seq.empty) + } + }.map(TweetMixerCandidate.interleave) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/curated_user_tls_per_language/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/curated_user_tls_per_language/BUILD.bazel new file mode 100644 index 000000000..ce44219dd --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/curated_user_tls_per_language/BUILD.bazel @@ -0,0 +1,12 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "servo/repo/src/main/scala", + "stitch/stitch-timelineservice", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/curated_user_tls_per_language/CuratedUserTlsPerLanguageCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/curated_user_tls_per_language/CuratedUserTlsPerLanguageCandidateSource.scala new file mode 100644 index 000000000..6072f087c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/curated_user_tls_per_language/CuratedUserTlsPerLanguageCandidateSource.scala @@ -0,0 +1,31 @@ +package com.twitter.tweet_mixer.candidate_source.curated_user_tls_per_language + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import com.twitter.stitch.timelineservice.TimelineService +import com.twitter.timelineservice.{thriftscala => tls} + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class CuratedUserTlsPerLanguageCandidateSource @Inject() (timelineService: TimelineService) + extends CandidateSource[Seq[tls.TimelineQuery], TweetCandidate] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "CuratedUserTlsPerLanguage") + override def apply(requests: Seq[tls.TimelineQuery]): Stitch[Seq[TweetCandidate]] = Stitch + .traverse(requests) { request => + timelineService.getTimeline(request).map { response => + response.entries.collect { + case tls.TimelineEntry.Tweet(tweet) => TweetCandidate(tweet.statusId) + } + } + }.map { + // Round-robin interleave across authors + _.filter(_.nonEmpty) // Remove empty lists + .transpose.flatten + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/BUILD.bazel new file mode 100644 index 000000000..bbee889c5 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/BUILD.bazel @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "haplolite-thrift/thrift/src/main/thrift:thrift-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "src/thrift/com/twitter/search:earlybird-scala", + "timelines/src/main/scala/com/twitter/timelines/clients/haplolite", + "timelineservice/common:model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/EarlybirdRealtimeCGTweetCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/EarlybirdRealtimeCGTweetCandidateSource.scala new file mode 100644 index 000000000..a7e37c078 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/EarlybirdRealtimeCGTweetCandidateSource.scala @@ -0,0 +1,104 @@ +package com.twitter.tweet_mixer.candidate_source.earlybird_realtime_cg + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.search.earlybird.{thriftscala => t} +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.model.ModuleNames.EarlybirdRealtimeCGEndpoint +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.timelines.clients.haplolite.TweetTimelineHaploCodec +import com.twitter.haplolite.{thriftscala => hl} +import com.twitter.timelineservice.model.Tweet +import com.twitter.timelineservice.model.core.TimelineKind + +@Singleton +class EarlybirdRealtimeCGTweetCandidateSource @Inject() ( + haploliteClient: hl.Haplolite.MethodPerEndpoint, + @Named(EarlybirdRealtimeCGEndpoint) earlybirdService: t.EarlybirdService.MethodPerEndpoint, + statsReceiver: StatsReceiver) + extends CandidateSource[InNetworkRequest, t.ThriftSearchResult] { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("EarlybirdRealtimeCGTweets") + + private val tweetTimelineHaploCodec = new TweetTimelineHaploCodec(statsReceiver) + + private val maxDepth = 2000 + + val haploliteSetSuccessCounter = statsReceiver.counter("HaploliteSetSuccess") + val haploliteSetFailureCounter = statsReceiver.counter("HaploliteSetFailure") + + private def convertToTimelineEntry(ebResult: t.ThriftSearchResult): Tweet = { + val id = ebResult.id + val userId = ebResult.metadata.map(_.fromUserId) + val isRetweet = ebResult.metadata.flatMap(_.isRetweet) + val isReply = ebResult.metadata.flatMap(_.isReply) + val sourceTweetId = + if (isRetweet.getOrElse(false)) + ebResult.metadata.map(_.sharedStatusId) + else None + val sourceUserId = + if (isRetweet.getOrElse(false)) + ebResult.metadata.map(_.referencedTweetAuthorId) + else None + val conversationId = + if (isReply.getOrElse(false)) + ebResult.metadata.flatMap(_.extraMetadata).flatMap(_.conversationId) + else None + val inReplyToTweetId = + if (isReply.getOrElse(false)) + ebResult.metadata.map(_.sharedStatusId) + else None + Tweet( + tweetId = id, + userId = userId, + sourceTweetId = sourceTweetId, + sourceUserId = sourceUserId, + conversationId = conversationId, + inReplyToTweetId = inReplyToTweetId) + } + + private def updateHaploCache(userId: Long, ebResults: Seq[t.ThriftSearchResult]) = { + val timelineEntries = ebResults.take(maxDepth).map(convertToTimelineEntry(_)) + val encodedEntries = tweetTimelineHaploCodec.encode(timelineEntries, TimelineKind.home) + val operation = hl.Operation( + Seq(hl.BulkKeys(hl.KeyNamespace.Home.value, Seq(userId))), + Some(hl.TimelineOperation(encodedEntries)) + ) + val setEntriesResult = haploliteClient.setEntries(hl.SetEntriesRequest(Seq(operation))) + setEntriesResult + .onSuccess { + case _ => haploliteSetSuccessCounter.incr() + }.onFailure { + case _ => haploliteSetFailureCounter.incr() + }.unit + } + + override def apply( + request: InNetworkRequest + ): Stitch[Seq[t.ThriftSearchResult]] = { + val ebRequest = request.earlybirdRequest + val excludedIds = request.excludedIds + val ebStitchResults = OffloadFuturePools.offloadFuture( + earlybirdService + .search(ebRequest) + .map { response: t.EarlybirdResponse => + response.searchResults + .map(_.results) + .getOrElse(Seq.empty) + }) + ebStitchResults.applyEffect { + case ebResults => + if (request.writeBackToHaplo) { + Stitch.callFuture( + updateHaploCache(ebRequest.searchQuery.searcherId.getOrElse(-1L), ebResults)) + } else { + Stitch.Unit + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/InNetworkRequest.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/InNetworkRequest.scala new file mode 100644 index 000000000..4ce91c8bb --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg/InNetworkRequest.scala @@ -0,0 +1,8 @@ +package com.twitter.tweet_mixer.candidate_source.earlybird_realtime_cg + +import com.twitter.search.earlybird.{thriftscala => t} + +case class InNetworkRequest( + earlybirdRequest: t.EarlybirdRequest, + excludedIds: Set[Long] = Set(), + writeBackToHaplo: Boolean = false) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/engaged_users/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/engaged_users/BUILD.bazel new file mode 100644 index 000000000..603e314ec --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/engaged_users/BUILD.bazel @@ -0,0 +1,12 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "src/thrift/com/twitter/twistly:twistly-scala", + "strato/config/columns/recommendations/twistly:twistly-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/engaged_users/RecentEngagedUsersCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/engaged_users/RecentEngagedUsersCandidateSource.scala new file mode 100644 index 000000000..ec21d2712 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/engaged_users/RecentEngagedUsersCandidateSource.scala @@ -0,0 +1,35 @@ +package com.twitter.tweet_mixer.candidate_source.engaged_users + +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyFetcherSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.recommendations.twistly.TweetRecentEngagedUsersClientColumn +import com.twitter.tweet_mixer.feature.EntityTypes.UserId +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class RecentEngagedUsersCandidateSource @Inject() ( + tweetRecentEngagedUsersClientColumn: TweetRecentEngagedUsersClientColumn) + extends StratoKeyFetcherSource[ + TweetRecentEngagedUsersClientColumn.Key, + TweetRecentEngagedUsersClientColumn.Value, + UserId + ] { + + override val fetcher: Fetcher[ + TweetRecentEngagedUsersClientColumn.Key, + Unit, + TweetRecentEngagedUsersClientColumn.Value + ] = + tweetRecentEngagedUsersClientColumn.fetcher + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "RecentEngagedUsers") + + override protected def stratoResultTransformer( + stratoResult: tweetRecentEngagedUsersClientColumn.Value + ): Seq[UserId] = { + stratoResult.recentEngagedUsers.map(_.userId) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/events/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/events/BUILD.bazel new file mode 100644 index 000000000..2942461c6 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/events/BUILD.bazel @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "servo/repo/src/main/scala", + "strato/config/columns/recommendations/simclusters_v2:simclusters_v2-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/events/EventsCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/events/EventsCandidateSource.scala new file mode 100644 index 000000000..463f12dc2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/events/EventsCandidateSource.scala @@ -0,0 +1,67 @@ +package com.twitter.tweet_mixer.candidate_source.events + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ExpiringLruInProcessCache +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Client +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.recommendations.simclusters_v2.TopPostsPerEventClientColumn +import com.twitter.tweet_mixer.candidate_source.events.EventsCandidateSource.cache +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.util.Random + +object EventsCandidateSource { + private val BaseTTL = 4 + private val TTL = (BaseTTL + Random.nextInt(3)).minutes + + val cache: InProcessCache[EventsRequest, Seq[TweetMixerCandidate]] = + new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 100) +} + +case class EventsRequest( + key: TopPostsPerEventClientColumn.Key, + irrelevanceDownrank: Double) + +@Singleton +class EventsCandidateSource @Inject() ( + @Named("StratoClientWithModerateTimeout") stratoClient: Client) + extends CandidateSource[ + EventsRequest, + TweetMixerCandidate + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("Events") + + private val fetcher: Fetcher[ + TopPostsPerEventClientColumn.Key, + Unit, + TopPostsPerEventClientColumn.Value + ] = new TopPostsPerEventClientColumn(stratoClient).fetcher + + override def apply( + request: EventsRequest + ): Stitch[Seq[TweetMixerCandidate]] = OffloadFuturePools.offloadStitch { + cache.get(request).map(Stitch.value(_)).getOrElse { + fetcher.fetch(request.key).map { result => + val posts = result.v.getOrElse(Seq.empty) + + val candidates = posts.map { post => + TweetMixerCandidate( + tweetId = post.tweetId, + score = if (post.isRelevant) post.score else post.score * request.irrelevanceDownrank, + seedId = 0L + ) + } + cache.set(request, candidates) + candidates + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/BUILD.bazel new file mode 100644 index 000000000..e9b74414c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/BUILD.bazel @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "servo/repo", + "strato/config/columns/evergreen_videos:evergreen_videos-strato-client", + "strato/config/columns/evergreen_videos/semantic_video:semantic_video-strato-client", + "strato/config/columns/evergreen_videos/twitter_clip_v0_long_video:twitter_clip_v0_long_video-strato-client", + "strato/config/columns/evergreen_videos/twitter_clip_v0_short_video:twitter_clip_v0_short_video-strato-client", + "strato/src/main/scala/com/twitter/strato/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/EvergreenVideosSearchByTweetQuery.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/EvergreenVideosSearchByTweetQuery.scala new file mode 100644 index 000000000..a5ed896e9 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/EvergreenVideosSearchByTweetQuery.scala @@ -0,0 +1,10 @@ +package com.twitter.tweet_mixer.candidate_source.evergreen_videos + +case class EvergreenVideosSearchByTweetQuery( + tweetIds: Seq[Long], + textMap: Option[Map[Long, String]] = None, + size: Int, + minWidth: Int, + minHeight: Int, + minDurationSec: Int, + maxDurationSec: Int) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/EvergreenVideosSearchByUserIdsQuery.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/EvergreenVideosSearchByUserIdsQuery.scala new file mode 100644 index 000000000..b8efb9aed --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/EvergreenVideosSearchByUserIdsQuery.scala @@ -0,0 +1,5 @@ +package com.twitter.tweet_mixer.candidate_source.evergreen_videos + +case class EvergreenVideosSearchByUserIdsQuery( + userIds: Seq[Long], + size: Int) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/HistoricalEvergreenVideosCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/HistoricalEvergreenVideosCandidateSource.scala new file mode 100644 index 000000000..38b923cb3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/HistoricalEvergreenVideosCandidateSource.scala @@ -0,0 +1,44 @@ +package com.twitter.tweet_mixer.candidate_source.evergreen_videos + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.stitch.Stitch +import com.twitter.strato.config.ElasticsearchPaginate +import com.twitter.strato.generated.client.evergreen_videos.HistoricalSearchByUserIdsClientColumn +import com.twitter.strato.generated.client.evergreen_videos.HistoricalSearchByUserIdsClientColumn.UserIdsAndSize +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HistoricalEvergreenVideosCandidateSource @Inject() ( + evergreenVideosHistoricalSearchByUserIdsClientColumn: HistoricalSearchByUserIdsClientColumn, + inputStatsReceiver: StatsReceiver) + extends CandidateSource[EvergreenVideosSearchByUserIdsQuery, TweetMixerCandidate] { + + private val scopedStats = inputStatsReceiver.scope(getClass.getSimpleName) + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("EvergreenVideos") + + override def apply( + request: EvergreenVideosSearchByUserIdsQuery + ): Stitch[Seq[TweetMixerCandidate]] = { + val paginator = evergreenVideosHistoricalSearchByUserIdsClientColumn.paginator + + val userIdsStr = request.userIds + .map { userId => + userId.toString + }.mkString("[", ",", "]") + + val beginCursor: HistoricalSearchByUserIdsClientColumn.Cursor = + ElasticsearchPaginate.Begin(UserIdsAndSize(userIds = userIdsStr, size = request.size)) + + paginator.paginate(beginCursor).map { page => + val tweets = page.data.map { idStr => + TweetMixerCandidate(tweetId = idStr.toLong, score = 0.0, seedId = -1L) + } + scopedStats.stat("tweetCount").add(tweets.size) + tweets + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/MemeVideoCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/MemeVideoCandidateSource.scala new file mode 100644 index 000000000..b35918b98 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/MemeVideoCandidateSource.scala @@ -0,0 +1,52 @@ +package com.twitter.tweet_mixer.candidate_source.evergreen_videos + +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Client +import com.twitter.strato.config.ElasticsearchPaginate +import com.twitter.strato.generated.client.evergreen_videos.HistoricalSearchByTextClientColumn +import com.twitter.strato.generated.client.evergreen_videos.HistoricalSearchByTextClientColumn.TextAndSize +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class MemeVideoCandidateSource @Inject() ( + @Named("StratoClientWithModerateTimeout") stratoClient: Client) + extends CandidateSource[PipelineQuery, TweetMixerCandidate] { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("MemeVideoCandidateSourceUsingSearchModule") + + override def apply( + request: PipelineQuery + ): Stitch[Seq[TweetMixerCandidate]] = { + + val funnyEmojis: String = + "\uD83D\uDE02\uD83E\uDD23\uD83D\uDE06" + "lang:" + request.getLanguageCode + val beginCursor: HistoricalSearchByTextClientColumn.Cursor = + ElasticsearchPaginate.Begin(TextAndSize(text = funnyEmojis, size = 200)) + + val paginator = + stratoClient.paginator[ + HistoricalSearchByTextClientColumn.Cursor, + HistoricalSearchByTextClientColumn.Page + ](HistoricalSearchByTextClientColumn.Path) + + OffloadFuturePools.offloadStitch { + paginator.paginate(beginCursor).map { page => + page.data.map { idStr => + TweetMixerCandidate( + tweetId = idStr.toLong, + score = 0.0, + seedId = -1L + ) + } + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/SemanticVideoCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/SemanticVideoCandidateSource.scala new file mode 100644 index 000000000..2ab67c33d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/SemanticVideoCandidateSource.scala @@ -0,0 +1,57 @@ +package com.twitter.tweet_mixer.candidate_source.evergreen_videos + +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.evergreen_videos.semantic_video.AnnOnTweetClientColumn +import com.twitter.strato.generated.client.evergreen_videos.semantic_video.AnnOnTweetClientColumn.IdAndScore +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SemanticVideoCandidateSource @Inject() ( + stratoColumn: AnnOnTweetClientColumn) + extends CandidateSource[EvergreenVideosSearchByTweetQuery, TweetMixerCandidate] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("SemanticVideo") + + private def isTextEligible(text: String): Boolean = { + val words = text.split("\\s+") + words.length >= 3 || text.length >= 10 + } + + override def apply( + request: EvergreenVideosSearchByTweetQuery + ): Stitch[Seq[TweetMixerCandidate]] = { + request.textMap + .map { textMap => + Stitch + .traverse(request.tweetIds) { tweetId => + val text = textMap.getOrElse(tweetId, "") + if (isTextEligible(text)) { + stratoColumn.fetcher.fetch(tweetId).map { response => + response.v + .map { relatedTweets: Seq[IdAndScore] => + relatedTweets.map(tweet => + TweetMixerCandidate( + tweet.id, + tweet.score, + tweetId + )) + }.getOrElse(Seq.empty) + } + } else { + Stitch.value(Seq.empty) + } + }.map { resultLists => + // round robin flatten + resultLists + .flatMap(list => list.zipWithIndex) + .sortBy(_._2) + .map(_._1) + .take(request.size) + } + }.getOrElse(Stitch.value(Seq.empty)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/TwitterClipV0LongVideoCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/TwitterClipV0LongVideoCandidateSource.scala new file mode 100644 index 000000000..bebdee0a1 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/TwitterClipV0LongVideoCandidateSource.scala @@ -0,0 +1,53 @@ +package com.twitter.tweet_mixer.candidate_source.evergreen_videos + +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.evergreen_videos.twitter_clip_v0_long_video.AnnSearchClientColumn +import com.twitter.strato.generated.client.evergreen_videos.twitter_clip_v0_long_video.AnnSearchClientColumn.IdAndScore +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TwitterClipV0LongVideoCandidateSource @Inject() ( + stratoColumn: AnnSearchClientColumn) + extends CandidateSource[EvergreenVideosSearchByTweetQuery, TweetMixerCandidate] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "TwitterClipV0LongVideo") + + override def apply( + request: EvergreenVideosSearchByTweetQuery + ): Stitch[Seq[TweetMixerCandidate]] = { + OffloadFuturePools + .offloadStitch { + Stitch + .traverse(request.tweetIds) { tweetId => + stratoColumn.fetcher + .fetch( + AnnSearchClientColumn.Key( + tweetId = tweetId, + size = Some(request.size) + )).map { response => + response.v + .map { relatedTweets: Seq[IdAndScore] => + relatedTweets.map(tweet => + TweetMixerCandidate( + tweet.id, + tweet.score, + tweetId + )) + }.getOrElse(Seq.empty) + } + } + }.map { resultLists => + // round robin flatten + resultLists + .flatMap(list => list.zipWithIndex) + .sortBy(_._2) + .map(_._1) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/TwitterClipV0ShortVideoCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/TwitterClipV0ShortVideoCandidateSource.scala new file mode 100644 index 000000000..8eca9f119 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos/TwitterClipV0ShortVideoCandidateSource.scala @@ -0,0 +1,54 @@ +package com.twitter.tweet_mixer.candidate_source.evergreen_videos + +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.evergreen_videos.twitter_clip_v0_short_video.AnnSearchClientColumn +import com.twitter.strato.generated.client.evergreen_videos.twitter_clip_v0_short_video.AnnSearchClientColumn.IdAndScore +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TwitterClipV0ShortVideoCandidateSource @Inject() ( + stratoColumn: AnnSearchClientColumn) + extends CandidateSource[EvergreenVideosSearchByTweetQuery, TweetMixerCandidate] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "TwitterClipV0ShortVideo") + + override def apply( + request: EvergreenVideosSearchByTweetQuery + ): Stitch[Seq[TweetMixerCandidate]] = { + OffloadFuturePools + .offloadStitch { + Stitch + .traverse(request.tweetIds) { tweetId => + stratoColumn.fetcher + .fetch( + AnnSearchClientColumn.Key( + tweetId = tweetId, + size = Some(request.size) + )).map { response => + response.v + .map { relatedTweets: Seq[IdAndScore] => + relatedTweets.map(tweet => + TweetMixerCandidate( + tweet.id, + tweet.score, + tweetId + )) + }.getOrElse(Seq.empty) + } + } + }.map { resultLists => + // round robin flatten + resultLists + .flatMap(list => list.zipWithIndex) + .sortBy(_._2) + .map(_._1) + } + + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/BUILD.bazel new file mode 100644 index 000000000..3acd712c3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/BUILD.bazel @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "hydra/common/libraries/src/main/scala/com/twitter/hydra/common/utils", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline", + "servo/repo", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + "vecdb/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DRANNKey.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DRANNKey.scala new file mode 100644 index 000000000..623c9c25c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DRANNKey.scala @@ -0,0 +1,37 @@ +package com.twitter.tweet_mixer.candidate_source.ndr_ann + +case class DRANNKey( + id: Long, + embedding: Option[Seq[Int]], + collectionName: String, + maxCandidates: Int, + scoreThreshold: Double = 0.0, + category: Option[String] = None, + enableGPU: Boolean = false, + isHighQuality: Option[Boolean] = None, + isLowNegEngRatio: Option[Boolean] = None, + tier: Option[String] = None, +) + +case class EmbeddingANNKey( + id: Long, + embedding: Option[Seq[Double]], + collectionName: String, + maxCandidates: Int, + scoreThreshold: Double = 0.0, + category: Option[String] = None, + enableGPU: Boolean = false, + isHighQuality: Option[Boolean] = None, + tier: Option[String] = None) + +case class ContentEmbeddingUserANNKey( + id: Long, + userId: Long, + embedding: Option[Seq[Double]], + collectionName: String, + maxCandidates: Int, + scoreThreshold: Double = 0.0, + category: Option[String] = None, + enableGPU: Boolean = false, + isHighQuality: Option[Boolean] = None, + tier: Option[String] = None) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DRMultipleANNQuery.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DRMultipleANNQuery.scala new file mode 100644 index 000000000..4fff995a0 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DRMultipleANNQuery.scala @@ -0,0 +1,13 @@ +package com.twitter.tweet_mixer.candidate_source.ndr_ann + +case class DRMultipleANNQuery( + annKeys: Seq[DRANNKey], + enableCache: Boolean) + +case class EmbeddingMultipleANNQuery( + annKeys: Seq[EmbeddingANNKey], + enableCache: Boolean) + +case class ContentEmbeddingMultipleUserANNQuery( + annKeys: Seq[ContentEmbeddingUserANNKey], + enableCache: Boolean) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalTweetTweetANNCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalTweetTweetANNCandidateSource.scala new file mode 100644 index 000000000..b93a89d35 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalTweetTweetANNCandidateSource.scala @@ -0,0 +1,116 @@ +package com.twitter.tweet_mixer.candidate_source.ndr_ann + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.utils.BucketSnowflakeIdAgeStats +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import com.twitter.tweet_mixer.utils.Utils.TweetId +import com.twitter.vecdb.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class DeepRetrievalTweetTweetANNCandidateSource @Inject() ( + @Named(ModuleNames.VecDBAnnServiceClient) + annClient: t.VecDB.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient, + statsReceiver: StatsReceiver) + extends MemcachedCandidateSource[ + DRMultipleANNQuery, + DRANNKey, + (TweetId, Double), + TweetMixerCandidate + ] { + + private val stats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetScoreStats = statsReceiver.stat("tweetScore") + private val tweetSizePerSignalStats = statsReceiver.stat("tweetSizePerSignal") + private val tweetAgeStats = + BucketSnowflakeIdAgeStats[TweetMixerCandidate]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _.tweetId)(stats.scope("tweetAge")) + + private val annStats = stats.scope("ANN") + private val annResultAgeStats = + BucketSnowflakeIdAgeStats[(TweetId, Double)]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _._1)(annStats) + private val annScoreStats = annStats.stat("score") + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("DeepRetrievalTweetTweetANNCandidateSource") + + override val TTL: Int = Utils.randomizedTTL(600) // 10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: DRANNKey): String = + f"dr_tweet_${key.collectionName}_${key.id.toString}_${key.scoreThreshold.toString}" + + val valueTransformer: Transformer[Seq[(TweetId, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + override def enableCache(request: DRMultipleANNQuery): Boolean = request.enableCache + + override def getKeys(request: DRMultipleANNQuery): Stitch[Seq[DRANNKey]] = + Stitch.value(request.annKeys) + + override def getCandidatesFromStore( + key: DRANNKey + ): Stitch[Seq[(TweetId, Double)]] = { + val futureResult = annClient + .searchById( + dataset = key.collectionName, + id = key.id, + params = Some( + t.SearchParams( + scoreThreshold = Some(key.scoreThreshold), + limit = Some(key.maxCandidates)))) + .map { response: t.SearchResponseV2 => + val result = response.points match { + case points: Seq[t.ScoredPointV2] => + points.map { point => + val score = point.score + annScoreStats.add(score.toFloat * 1000) + (point.id, score) + } + case _ => + Seq.empty + } + annResultAgeStats.count(result) + result + } + Stitch.callFuture(futureResult) + } + + override def postProcess( + request: DRMultipleANNQuery, + keys: Seq[DRANNKey], + resultsSeq: Seq[Seq[(TweetId, Double)]] + ): Seq[TweetMixerCandidate] = { + val candidates = keys.zip(resultsSeq).map { + case (key, results) => + results + .map { + case (id, score) => + tweetScoreStats.add(score.toFloat * 1000) + TweetMixerCandidate(id, score, key.id) + } + } + candidates.foreach { seq => + tweetSizePerSignalStats.add(seq.size) + } + val result = TweetMixerCandidate.interleave(candidates) + tweetAgeStats.count(result) + result + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalTweetTweetEmbeddingANNCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalTweetTweetEmbeddingANNCandidateSource.scala new file mode 100644 index 000000000..2a67299b7 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalTweetTweetEmbeddingANNCandidateSource.scala @@ -0,0 +1,175 @@ +package com.twitter.tweet_mixer.candidate_source.ndr_ann + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hydra.common.utils.{Utils => HydraUtils} +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.module.GPURetrievalHttpClient +import com.twitter.tweet_mixer.utils.BucketSnowflakeIdAgeStats +import com.twitter.tweet_mixer.utils.BucketSnowflakeIdAgeStats.MillisecondsPerHour +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import com.twitter.tweet_mixer.utils.Utils.TweetId +import com.twitter.util.Future +import com.twitter.vecdb.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +class DeepRetrievalTweetTweetEmbeddingANNCandidateSource( + annClient: t.VecDB.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient, + gpuRetrievalHttpClient: GPURetrievalHttpClient, + statsReceiver: StatsReceiver, + sourceIdentifier: String) + extends MemcachedCandidateSource[ + DRMultipleANNQuery, + DRANNKey, + (TweetId, Double), + TweetMixerCandidate + ] { + + private val stats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetScoreStats = stats.stat("tweetScore") + private val tweetSizePerSignalStats = stats.stat("tweetSizePerSignal") + private val tweetAgeScope = stats.scope("tweetAge") + private val tweetAgeStats = + BucketSnowflakeIdAgeStats[TweetMixerCandidate](MillisecondsPerHour, _.tweetId)(tweetAgeScope) + private val annStats = stats.scope("ANN") + private val annResultAgeStats = + BucketSnowflakeIdAgeStats[(TweetId, Double)](MillisecondsPerHour, _._1)(annStats) + private val annScoreStats = annStats.stat("score") + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier(sourceIdentifier) + + override val TTL: Int = Utils.randomizedTTL(180) // 3 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: DRANNKey): String = + f"${sourceIdentifier}_${key.collectionName}_${key.tier.getOrElse("")}_${key.id.toString}_${key.scoreThreshold.toString}" + + val valueTransformer: Transformer[Seq[(TweetId, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + override def enableCache(request: DRMultipleANNQuery): Boolean = request.enableCache + + override def getKeys(request: DRMultipleANNQuery): Stitch[Seq[DRANNKey]] = + Stitch.value(request.annKeys) + + private def getCandidatesFromVecDB( + embedding: Seq[Int], + collectionName: String, + maxCandidates: Int, + scoreThreshold: Double, + filter: Option[t.Filter] + ) = { + annClient + .search( + dataset = collectionName, + vector = HydraUtils.intBitsSeqToFloatSeq(embedding).map(_.toDouble), + params = + Some(t.SearchParams(scoreThreshold = Some(scoreThreshold), limit = Some(maxCandidates))), + filter = filter + ).map { response: t.SearchResponse => + response.points match { + case points: Seq[t.ScoredPoint] => + points.map { point => + (point.id, point.score) + } + case _ => + Seq.empty + } + } + } + + private def getCandidatesFromGPU(embedding: Seq[Int]) = { + gpuRetrievalHttpClient.getNeighbors(embedding) + } + + override def getCandidatesFromStore( + key: DRANNKey + ): Stitch[Seq[(TweetId, Double)]] = { + val tier = key.tier.map { tier => + t.FieldPredicate(key = t.Key.PayloadField("tier"), condition = t.ValueCondition.EqStr(tier)) + } + + val filters = tier.toSeq + val filter = if (filters.nonEmpty) { + Some(t.Filter.All(filters)) + } else None + val futureResult = for { + response <- key.embedding match { + case Some(embedding) => + if (key.enableGPU) getCandidatesFromGPU(embedding) + else + getCandidatesFromVecDB( + embedding, + key.collectionName, + key.maxCandidates, + key.scoreThreshold, + filter) + case None => + Future.Nil + } + } yield { + val result = response.map { + case (tweetId, score) => + annScoreStats.add(score.toFloat * 1000) + (tweetId, score) + } + annResultAgeStats.count(result) + result + } + Stitch.callFuture(futureResult) + } + + override def postProcess( + request: DRMultipleANNQuery, + keys: Seq[DRANNKey], + resultsSeq: Seq[Seq[(TweetId, Double)]] + ): Seq[TweetMixerCandidate] = { + val candidates = keys.zip(resultsSeq).map { + case (key, results) => + results + .map { + case (id, score) => + tweetScoreStats.add(score.toFloat * 1000) + TweetMixerCandidate(id, score, key.id) + } + } + candidates.foreach { seq => + tweetSizePerSignalStats.add(seq.size) + } + val result = TweetMixerCandidate.interleave(candidates) + tweetAgeStats.count(result) + result + } +} + +@Singleton +class DeepRetrievalTweetTweetEmbeddingANNCandidateSourceFactory @Inject() ( + @Named(ModuleNames.VecDBAnnServiceClient) + annClient: t.VecDB.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient, + @Named(ModuleNames.GPURetrievalProdHttpClient) + gpuRetrievalHttpClient: GPURetrievalHttpClient, + statsReceiver: StatsReceiver) { + + def build(identifier: String): DeepRetrievalTweetTweetEmbeddingANNCandidateSource = { + new DeepRetrievalTweetTweetEmbeddingANNCandidateSource( + annClient, + memcacheClient, + gpuRetrievalHttpClient, + statsReceiver, + identifier + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalUserTweetANNCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalUserTweetANNCandidateSource.scala new file mode 100644 index 000000000..a58b90cc7 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/DeepRetrievalUserTweetANNCandidateSource.scala @@ -0,0 +1,193 @@ +package com.twitter.tweet_mixer.candidate_source.ndr_ann + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hydra.common.utils.{Utils => HydraUtils} +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.module.GPURetrievalHttpClient +import com.twitter.tweet_mixer.utils.BucketSnowflakeIdAgeStats +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import com.twitter.tweet_mixer.utils.Utils.TweetId +import com.twitter.util.Future +import com.twitter.vecdb.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +class DeepRetrievalUserTweetANNCandidateSource( + annClient: t.VecDB.MethodPerEndpoint, + gpuRetrievalHttpClient: GPURetrievalHttpClient, + memcacheClient: MemcacheStitchClient, + statsReceiver: StatsReceiver, + sourceIdentifier: String) + extends MemcachedCandidateSource[ + DRMultipleANNQuery, + DRANNKey, + (TweetId, Double), + TweetMixerCandidate + ] { + + private val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetScoreStats = scopedStats.stat("tweetScore") + private val tweetSizePerSignalStats = scopedStats.stat("tweetSizePerSignal") + private val tweetAgeStats = + BucketSnowflakeIdAgeStats[TweetMixerCandidate]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _.tweetId)(scopedStats.scope("tweetAge")) + + private val annStats = scopedStats.scope("ANN") + private val annResultAgeStats = + BucketSnowflakeIdAgeStats[(TweetId, Double)]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _._1)(annStats) + private val annScoreStats = annStats.stat("score") + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier(sourceIdentifier) + + override val TTL: Int = Utils.randomizedTTL(600) // 10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: DRANNKey): String = + f"${sourceIdentifier}_${key.collectionName}_${key.id.toString}_${key.scoreThreshold.toString}" + + val valueTransformer: Transformer[Seq[(TweetId, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + override def enableCache(request: DRMultipleANNQuery): Boolean = request.enableCache + + override def getKeys(request: DRMultipleANNQuery): Stitch[Seq[DRANNKey]] = + Stitch.value(request.annKeys) + + private def getCandidatesFromVecDB( + embedding: Seq[Int], + collectionName: String, + maxCandidates: Int, + scoreThreshold: Double, + filter: Option[t.Filter] + ) = { + annClient + .search( + dataset = collectionName, + vector = HydraUtils.intBitsSeqToFloatSeq(embedding).map(_.toDouble), + params = + Some(t.SearchParams(scoreThreshold = Some(scoreThreshold), limit = Some(maxCandidates))), + filter = filter + ).map { response: t.SearchResponse => + response.points match { + case points: Seq[t.ScoredPoint] => + points.map { point => + (point.id, point.score) + } + case _ => + Seq.empty + } + } + } + + private def getCandidatesFromGPU(embedding: Seq[Int]) = { + gpuRetrievalHttpClient.getNeighbors(embedding) + } + + override def getCandidatesFromStore( + key: DRANNKey + ): Stitch[Seq[(TweetId, Double)]] = { + val categoryFilter = key.category.map { cat => + t.FieldPredicate( + key = t.Key.PayloadField("category"), + condition = t.ValueCondition.EqStr(cat)) + } + val highQualityFilter = key.isHighQuality.map { isHighQuality => + t.FieldPredicate( + key = t.Key.PayloadField("isHighQuality"), + condition = t.ValueCondition.EqBool(isHighQuality)) + } + val lowNegEngRatioFilter = key.isLowNegEngRatio.map { isLowNegEngRatio => + t.FieldPredicate( + key = t.Key.PayloadField("isHighNegEngRatio"), + condition = t.ValueCondition.EqBool(isLowNegEngRatio)) + } + val tierFilter = key.tier.map { tier => + t.FieldPredicate(key = t.Key.PayloadField("tier"), condition = t.ValueCondition.EqStr(tier)) + } + val filters = categoryFilter.toSeq ++ highQualityFilter ++ lowNegEngRatioFilter ++ tierFilter + val filter = if (filters.nonEmpty) { + Some(t.Filter.All(filters)) + } else None + + val futureResult = for { + response <- key.embedding match { + case Some(embedding) => + if (key.enableGPU) getCandidatesFromGPU(embedding) + else + getCandidatesFromVecDB( + embedding, + key.collectionName, + key.maxCandidates, + key.scoreThreshold, + filter) + case None => + Future.Nil + } + } yield { + val result = response.map { + case (tweetId, score) => + annScoreStats.add(score.toFloat * 1000) + (tweetId, score) + } + annResultAgeStats.count(result) + result + } + Stitch.callFuture(futureResult) + } + + override def postProcess( + request: DRMultipleANNQuery, + keys: Seq[DRANNKey], + resultsSeq: Seq[Seq[(TweetId, Double)]] + ): Seq[TweetMixerCandidate] = { + val candidates = keys.zip(resultsSeq).map { + case (key, results) => + results + .map { + case (id, score) => + tweetScoreStats.add(score.toFloat * 1000) + TweetMixerCandidate(id, score, key.id) + } + } + candidates.foreach { seq => + tweetSizePerSignalStats.add(seq.size) + } + val result = TweetMixerCandidate.interleave(candidates) + tweetAgeStats.count(result) + result + } +} + +@Singleton +class DeepRetrievalUserTweetANNCandidateSourceFactory @Inject() ( + @Named(ModuleNames.VecDBAnnServiceClient) + annClient: t.VecDB.MethodPerEndpoint, + @Named(ModuleNames.GPURetrievalProdHttpClient) + gpuRetrievalHttpClient: GPURetrievalHttpClient, + memcacheClient: MemcacheStitchClient, + statsReceiver: StatsReceiver) { + + def build(identifier: String): DeepRetrievalUserTweetANNCandidateSource = { + new DeepRetrievalUserTweetANNCandidateSource( + annClient, + gpuRetrievalHttpClient, + memcacheClient, + statsReceiver, + identifier + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/EmbeddingANNCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/EmbeddingANNCandidateSource.scala new file mode 100644 index 000000000..b141d3c23 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/EmbeddingANNCandidateSource.scala @@ -0,0 +1,168 @@ +package com.twitter.tweet_mixer.candidate_source.ndr_ann + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.utils.BucketSnowflakeIdAgeStats +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import com.twitter.tweet_mixer.utils.Utils.TweetId +import com.twitter.util.Future +import com.twitter.vecdb.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +class EmbeddingANNCandidateSource( + annClient: t.VecDB.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient, + statsReceiver: StatsReceiver, + sourceIdentifier: String) + extends MemcachedCandidateSource[ + EmbeddingMultipleANNQuery, + EmbeddingANNKey, + (TweetId, Double), + TweetMixerCandidate + ] { + + private val stats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetScoreStats = stats.stat("tweetScore") + private val tweetSizePerSignalStats = stats.stat("tweetSizePerSignal") + private val tweetAgeStats = + BucketSnowflakeIdAgeStats[TweetMixerCandidate]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _.tweetId)(stats.scope("tweetAge")) + + private val annStats = stats.scope("EmbeddingANN") + private val annResultAgeStats = + BucketSnowflakeIdAgeStats[(TweetId, Double)]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _._1)(annStats) + private val annScoreStats = annStats.stat("score") + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier(sourceIdentifier) + + override val TTL: Int = Utils.randomizedTTL(600) // 10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: EmbeddingANNKey): String = + f"${sourceIdentifier}_${key.collectionName}_${key.id.toString}_${key.scoreThreshold.toString}" + + override def enableCache(request: EmbeddingMultipleANNQuery): Boolean = request.enableCache + + override def getKeys( + request: EmbeddingMultipleANNQuery + ): Stitch[Seq[EmbeddingANNKey]] = Stitch.value(request.annKeys) + + override val valueTransformer: Transformer[Seq[(TweetId, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + private def getCandidatesFromVecDB( + embedding: Seq[Double], + collectionName: String, + maxCandidates: Int, + scoreThreshold: Double, + filter: Option[t.Filter] + ) = { + annClient + .search( + dataset = collectionName, + vector = embedding, + params = + Some(t.SearchParams(scoreThreshold = Some(scoreThreshold), limit = Some(maxCandidates))), + filter = filter + ).map { response: t.SearchResponse => + response.points match { + case points: Seq[t.ScoredPoint] => + points.map { point => + (point.id, point.score) + } + case _ => + Seq.empty + } + } + } + + override def getCandidatesFromStore( + key: EmbeddingANNKey + ): Stitch[Seq[(TweetId, Double)]] = { + val tier = key.tier.map { tier => + t.FieldPredicate(key = t.Key.PayloadField("tier"), condition = t.ValueCondition.EqStr(tier)) + } + + val filters = tier.toSeq + val filter = if (filters.nonEmpty) { + Some(t.Filter.All(filters)) + } else None + + val futureResult = for { + response <- key.embedding match { + case Some(embedding) => + getCandidatesFromVecDB( + embedding, + key.collectionName, + key.maxCandidates, + key.scoreThreshold, + filter) + case None => + Future.Nil + } + } yield { + val result = response.map { + case (tweetId, score) => + annScoreStats.add(score.toFloat * 1000) + (tweetId, score) + } + annResultAgeStats.count(result) + result + } + Stitch.callFuture(futureResult) + } + + override def postProcess( + request: EmbeddingMultipleANNQuery, + keys: Seq[EmbeddingANNKey], + resultsSeq: Seq[Seq[(TweetId, Double)]] + ): Seq[TweetMixerCandidate] = { + val candidates = keys.zip(resultsSeq).map { + case (key, results) => + results + .map { + case (id, score) => + tweetScoreStats.add(score.toFloat * 1000) + TweetMixerCandidate(id, score, key.id) + } + } + candidates.foreach { seq => + tweetSizePerSignalStats.add(seq.size) + } + val result = TweetMixerCandidate.interleave(candidates) + tweetAgeStats.count(result) + result + } +} + +@Singleton +class EmbeddingANNCandidateSourceFactory @Inject() ( + @Named(ModuleNames.VecDBAnnServiceClient) + annClient: t.VecDB.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient, + statsReceiver: StatsReceiver) { + + def build(identifier: String): EmbeddingANNCandidateSource = { + new EmbeddingANNCandidateSource( + annClient, + memcacheClient, + statsReceiver, + identifier + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/MediaDeepRetrievalUserTweetANNCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/MediaDeepRetrievalUserTweetANNCandidateSource.scala new file mode 100644 index 000000000..a16956ea1 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/MediaDeepRetrievalUserTweetANNCandidateSource.scala @@ -0,0 +1,166 @@ +package com.twitter.tweet_mixer.candidate_source.ndr_ann + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hydra.common.utils.{Utils => HydraUtils} +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.utils.BucketSnowflakeIdAgeStats +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import com.twitter.tweet_mixer.utils.Utils.TweetId +import com.twitter.util.Future +import com.twitter.vecdb.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class MediaDeepRetrievalUserTweetANNCandidateSource @Inject() ( + @Named(ModuleNames.VecDBAnnServiceClient) + annClient: t.VecDB.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient, + statsReceiver: StatsReceiver, + sourceIdentifier: String) + extends MemcachedCandidateSource[ + DRMultipleANNQuery, + DRANNKey, + (TweetId, Double), + TweetMixerCandidate + ] { + + private val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetScoreStats = scopedStats.stat("tweetScore") + private val tweetSizePerSignalStats = scopedStats.stat("tweetSizePerSignal") + private val tweetAgeStats = + BucketSnowflakeIdAgeStats[TweetMixerCandidate]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _.tweetId)(scopedStats.scope("tweetAge")) + + private val annStats = scopedStats.scope("ANN") + private val annResultAgeStats = + BucketSnowflakeIdAgeStats[(TweetId, Double)]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _._1)(annStats) + private val annScoreStats = annStats.stat("score") + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier(sourceIdentifier) + + override val TTL: Int = Utils.randomizedTTL(600) // 10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: DRANNKey): String = + f"media_dr_user_${key.collectionName}_${key.id.toString}" + + val valueTransformer: Transformer[Seq[(TweetId, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + override def enableCache(request: DRMultipleANNQuery): Boolean = request.enableCache + + override def getKeys(request: DRMultipleANNQuery): Stitch[Seq[DRANNKey]] = { + Stitch.value(request.annKeys) + } + + override def getCandidatesFromStore( + key: DRANNKey + ): Stitch[Seq[(TweetId, Double)]] = { + val categoryFilter = key.category.map { cat => + t.FieldPredicate( + key = t.Key.PayloadField("category"), + condition = t.ValueCondition.EqStr(cat)) + } + val highQualityFilter = key.isHighQuality.map { isHighQuality => + t.FieldPredicate( + key = t.Key.PayloadField("isHighQuality"), + condition = t.ValueCondition.EqBool(isHighQuality)) + } + val lowNegEngRatioFilter = key.isLowNegEngRatio.map { isLowNegEngRatio => + t.FieldPredicate( + key = t.Key.PayloadField("isHighNegEngRatio"), + condition = t.ValueCondition.EqBool(isLowNegEngRatio)) + } + val filters = categoryFilter.toSeq ++ highQualityFilter ++ lowNegEngRatioFilter + val filter = if (filters.nonEmpty) { + Some(t.Filter.All(filters)) + } else None + + val futureResult = for { + response <- key.embedding match { + case Some(embedding) => + annClient + .search( + dataset = key.collectionName, + vector = HydraUtils.intBitsSeqToFloatSeq(embedding).map(_.toDouble), + params = + Some(t.SearchParams(limit = Some(key.maxCandidates), includePayload = Some(true))), + filter = filter + ).map { response: t.SearchResponse => + response.points match { + case points: Seq[t.ScoredPoint] => + points.map { point => + (point.id, point.score) + } + case _ => + Seq.empty + } + } + case None => + Future.Nil + } + } yield { + val result = response.map { + case (tweetId, score) => + annScoreStats.add(score.toFloat * 1000) + (tweetId, score) + } + annResultAgeStats.count(result) + result + } + Stitch.callFuture(futureResult) + } + + override def postProcess( + request: DRMultipleANNQuery, + keys: Seq[DRANNKey], + resultsSeq: Seq[Seq[(TweetId, Double)]] + ): Seq[TweetMixerCandidate] = { + val candidates = keys.zip(resultsSeq).map { + case (key, results) => + results + .map { + case (id, score) => + tweetScoreStats.add(score.toFloat * 1000) + TweetMixerCandidate(id, score, key.id) + } + } + candidates.foreach { seq => + tweetSizePerSignalStats.add(seq.size) + } + val result = TweetMixerCandidate.interleave(candidates) + tweetAgeStats.count(result) + result + } +} + +@Singleton +class MediaDeepRetrievalUserTweetANNCandidateSourceFactory @Inject() ( + @Named(ModuleNames.VecDBAnnServiceClient) + annClient: t.VecDB.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient, + statsReceiver: StatsReceiver) { + + def build(identifier: String): MediaDeepRetrievalUserTweetANNCandidateSource = { + new MediaDeepRetrievalUserTweetANNCandidateSource( + annClient, + memcacheClient, + statsReceiver, + identifier + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/UserInterestANNCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/UserInterestANNCandidateSource.scala new file mode 100644 index 000000000..a4892da50 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann/UserInterestANNCandidateSource.scala @@ -0,0 +1,174 @@ +package com.twitter.tweet_mixer.candidate_source.ndr_ann + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.servo.util.Transformer +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.module.GPURetrievalHttpClient +import com.twitter.tweet_mixer.utils.BucketSnowflakeIdAgeStats +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import com.twitter.tweet_mixer.utils.Utils.TweetId +import com.twitter.util.Future +import com.twitter.vecdb.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +class UserInterestEmbeddingANNCandidateSource( + annClient: t.VecDB.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient, + gpuRetrievalHttpClient: GPURetrievalHttpClient, + statsReceiver: StatsReceiver, + sourceIdentifier: String) + extends MemcachedCandidateSource[ + ContentEmbeddingMultipleUserANNQuery, + ContentEmbeddingUserANNKey, + (TweetId, Double), + TweetMixerCandidate + ] { + + private val stats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetScoreStats = stats.stat("tweetScore") + private val tweetSizePerSignalStats = stats.stat("tweetSizePerSignal") + private val tweetAgeStats = + BucketSnowflakeIdAgeStats[TweetMixerCandidate]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _.tweetId)(stats.scope("tweetAge")) + + private val annStats = stats.scope("ContentEmbeddingUserANN") + private val annResultAgeStats = + BucketSnowflakeIdAgeStats[(TweetId, Double)]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _._1)(annStats) + private val annScoreStats = annStats.stat("score") + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier(sourceIdentifier) + + override val TTL: Int = Utils.randomizedTTL(600) // 10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: ContentEmbeddingUserANNKey): String = + f"${sourceIdentifier}_${key.collectionName}_${key.id.toString}_${key.userId.toString}_${key.scoreThreshold.toString}" + + override def enableCache(request: ContentEmbeddingMultipleUserANNQuery): Boolean = request.enableCache + + override def getKeys( + request: ContentEmbeddingMultipleUserANNQuery + ): Stitch[Seq[ContentEmbeddingUserANNKey]] = + Stitch.value(request.annKeys) + + override val valueTransformer: Transformer[Seq[(TweetId, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + private def getCandidatesFromVecDB( + embedding: Seq[Double], + collectionName: String, + maxCandidates: Int, + scoreThreshold: Double, + filter: Option[t.Filter] + ) = { + annClient + .search( + dataset = collectionName, + vector = embedding, + params = + Some(t.SearchParams(scoreThreshold = Some(scoreThreshold), limit = Some(maxCandidates))), + filter = filter + ).map { response: t.SearchResponse => + response.points match { + case points: Seq[t.ScoredPoint] => + points.map { point => + (point.id, point.score) + } + case _ => + Seq.empty + } + } + } + + override def getCandidatesFromStore( + key: ContentEmbeddingUserANNKey + ): Stitch[Seq[(TweetId, Double)]] = { + val tier = key.tier.map { tier => + t.FieldPredicate(key = t.Key.PayloadField("tier"), condition = t.ValueCondition.EqStr(tier)) + } + + val filters = tier.toSeq + val filter = if (filters.nonEmpty) { + Some(t.Filter.All(filters)) + } else None + + val futureResult = for { + response <- key.embedding match { + case Some(embedding) => + getCandidatesFromVecDB( + embedding, + key.collectionName, + key.maxCandidates, + key.scoreThreshold, + filter) + case None => + Future.Nil + } + } yield { + val result = response.map { + case (tweetId, score) => + annScoreStats.add(score.toFloat * 1000) + (tweetId, score) + } + annResultAgeStats.count(result) + result + } + Stitch.callFuture(futureResult) + } + + override def postProcess( + request: ContentEmbeddingMultipleUserANNQuery, + keys: Seq[ContentEmbeddingUserANNKey], + resultsSeq: Seq[Seq[(TweetId, Double)]] + ): Seq[TweetMixerCandidate] = { + val candidates = keys.zip(resultsSeq).map { + case (key, results) => + results + .map { + case (id, score) => + tweetScoreStats.add(score.toFloat * 1000) + TweetMixerCandidate(id, score, key.id) + } + } + candidates.foreach { seq => + tweetSizePerSignalStats.add(seq.size) + } + val result = TweetMixerCandidate.interleave(candidates) + tweetAgeStats.count(result) + result + } +} + +@Singleton +class UserInterestEmbeddingANNCandidateSourceFactory @Inject() ( + @Named(ModuleNames.VecDBAnnServiceClient) + annClient: t.VecDB.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient, + @Named(ModuleNames.GPURetrievalProdHttpClient) + gpuRetrievalHttpClient: GPURetrievalHttpClient, + statsReceiver: StatsReceiver) { + + def build(identifier: String): UserInterestEmbeddingANNCandidateSource = { + new UserInterestEmbeddingANNCandidateSource( + annClient, + memcacheClient, + gpuRetrievalHttpClient, + statsReceiver, + identifier + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/BUILD.bazel new file mode 100644 index 000000000..436bd309d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/BUILD.bazel @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "servo/repo", + "src/thrift/com/twitter/trends/trip_v1:trip-tweets-thrift-scala", + "strato/config/columns/trends/trip:trip-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/PopularGeoTweetsCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/PopularGeoTweetsCandidateSource.scala new file mode 100644 index 000000000..64b697723 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/PopularGeoTweetsCandidateSource.scala @@ -0,0 +1,72 @@ +package com.twitter.tweet_mixer.candidate_source.popular_geo_tweets + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.trends.trip.TripTweetsProdClientColumn +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import com.twitter.trends.trip_v1.trip_tweets.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class PopularGeoTweetsCandidateSource @Inject() ( + tripStratoColumn: TripTweetsProdClientColumn, + inputStatsReceiver: StatsReceiver) + extends CandidateSource[TripStratoGeoQuery, t.TripTweet] { + + private val scopedStats = inputStatsReceiver.scope(getClass.getSimpleName) + private val emptyDomainCounter = scopedStats.counter("emptyDomain") + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("PopularGeoTweets") + private val unifiedTripTweetScore: Double = 0D + + override def apply(request: TripStratoGeoQuery): Stitch[Seq[t.TripTweet]] = { + if (request.domains.isEmpty) emptyDomainCounter.incr() + + getTweetsSortedByDomain( + domains = request.domains, + maxCandidatesPerSource = request.maxCandidatesPerSource, + maxPopGeoCandidates = request.maxPopGeoCandidates + ) + } + + private def getTweetsSortedByDomain( + domains: Seq[TripDomain], + maxCandidatesPerSource: Int, + maxPopGeoCandidates: Int + ): Stitch[Seq[t.TripTweet]] = OffloadFuturePools.offloadStitch { + val fetcher = tripStratoColumn.fetcher + + Stitch + .collect(domains.zipWithIndex.map { + case (tripDomain: TripDomain, domainIndex: Int) => + fetcher.fetch(tripDomain).map { + stratoValue => + val tripTweetsSeq = stratoValue.v + .map { response => + response.tweets + .take(maxCandidatesPerSource) + .zipWithIndex.map { + case (tripTweet, tweetIndex) => + val plainTripTweet = + t.TripTweet( + tweetId = tripTweet.tweetId, + score = unifiedTripTweetScore + ) + (tweetIndex, domainIndex, plainTripTweet) + } + }.getOrElse(Seq.empty) + + scopedStats.counter(tripDomain.sourceId).incr(tripTweetsSeq.size) + tripTweetsSeq + } + }).map(_.flatten).map { tweetsWithIndicesSeq: Seq[(Int, Int, t.TripTweet)] => + // sort by tweetIndex first, then domainIndex, + // so each domain (sourceId + others) can have equal weights being selected + tweetsWithIndicesSeq + .sortBy(t => (t._1, t._2)).map(_._3).distinct.take(maxPopGeoCandidates) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/TripStratoGeoQuery.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/TripStratoGeoQuery.scala new file mode 100644 index 000000000..76b276e23 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets/TripStratoGeoQuery.scala @@ -0,0 +1,8 @@ +package com.twitter.tweet_mixer.candidate_source.popular_geo_tweets + +import com.twitter.trends.trip_v1.trip_tweets.{thriftscala => t} + +case class TripStratoGeoQuery( + domains: Seq[t.TripDomain], + maxCandidatesPerSource: Int, + maxPopGeoCandidates: Int) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/BUILD.bazel new file mode 100644 index 000000000..4124d7474 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/BUILD.bazel @@ -0,0 +1,14 @@ +scala_library( + compiler_option_sets = ["fatal_warnings"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "servo/repo", + "strato/config/columns/trends/trip:trip-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + ], + sources = ["*.scala"], + strict_deps = True, + tags = ["bazel-compatible"], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/GrokTopicTweetsQuery.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/GrokTopicTweetsQuery.scala new file mode 100644 index 000000000..a84c8074b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/GrokTopicTweetsQuery.scala @@ -0,0 +1,7 @@ +package com.twitter.tweet_mixer.candidate_source.popular_grok_topic_tweets + +case class GrokTopicTweetsQuery( + userId: Long, + language: Option[String], + placeId: Option[Long], + maxNumCandidates: Int) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/PopGrokTopicTweetsCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/PopGrokTopicTweetsCandidateSource.scala new file mode 100644 index 000000000..5a518820a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets/PopGrokTopicTweetsCandidateSource.scala @@ -0,0 +1,34 @@ +package com.twitter.tweet_mixer.candidate_source.popular_grok_topic_tweets + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.trends.trip.GrokTopicTweetRecommendationsClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class PopGrokTopicTweetsCandidateSource @Inject() ( + grokTopicTweetRecommendationsClientColumn: GrokTopicTweetRecommendationsClientColumn) + extends CandidateSource[GrokTopicTweetsQuery, TweetCandidate] { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("PopGrokTopicTweets") + + private val fetcher = grokTopicTweetRecommendationsClientColumn.fetcher + + override def apply(request: GrokTopicTweetsQuery): Stitch[Seq[TweetCandidate]] = + OffloadFuturePools.offloadStitch { + val key = GrokTopicTweetRecommendationsClientColumn.Key( + userId = request.userId, + language = request.language, + placeId = request.placeId + ) + fetcher.fetch(key, {}).map { response => + response.v.map(_.map(TweetCandidate(_)).take(request.maxNumCandidates)) + .getOrElse(Seq.empty) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/BUILD.bazel new file mode 100644 index 000000000..c60ef8e9a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/BUILD.bazel @@ -0,0 +1,14 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "servo/repo", + "src/thrift/com/twitter/trends/trip_v1:trip-tweets-thrift-scala", + "strato/config/columns/trends/trip:trip-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/PopularTopicTweetsCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/PopularTopicTweetsCandidateSource.scala new file mode 100644 index 000000000..25236a797 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/PopularTopicTweetsCandidateSource.scala @@ -0,0 +1,74 @@ +package com.twitter.tweet_mixer.candidate_source.popular_topic_tweets + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.trends.trip.TripTweetsProdClientColumn +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import com.twitter.trends.trip_v1.trip_tweets.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class PopularTopicTweetsCandidateSource @Inject() ( + tripStratoColumn: TripTweetsProdClientColumn, + inputStatsReceiver: StatsReceiver) + extends CandidateSource[TripStratoTopicQuery, t.TripTweet] { + + private val scopedStats = inputStatsReceiver.scope(getClass.getSimpleName) + private val emptyDomainCounter = scopedStats.counter("emptyDomain") + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "PopularTopicTweets") + + private val unifiedTripTweetScore: Double = 0D + + override def apply(request: TripStratoTopicQuery): Stitch[Seq[t.TripTweet]] = { + if (request.domains.isEmpty) emptyDomainCounter.incr() + + getTweetsSortedByDomain( + domains = request.domains, + maxCandidatesPerSource = request.maxCandidatesPerSource, + maxPopTopicCandidates = request.maxPopTopicCandidates + ) + } + + private def getTweetsSortedByDomain( + domains: Seq[TripDomain], + maxCandidatesPerSource: Int, + maxPopTopicCandidates: Int + ): Stitch[Seq[t.TripTweet]] = { + val fetcher = tripStratoColumn.fetcher + + Stitch + .collect(domains.zipWithIndex.map { + case (tripDomain: TripDomain, domainIndex: Int) => + fetcher.fetch(tripDomain).map { + stratoValue => + val tripTweetsSeq = stratoValue.v + .map { response => + response.tweets + .take(maxCandidatesPerSource) + .zipWithIndex.map { + case (tripTweet, tweetIndex) => + // score was the rank of a tweet in a source; not comparable across domains + val plainTripTweet = + t.TripTweet( + tweetId = tripTweet.tweetId, + score = unifiedTripTweetScore + ) + (tweetIndex, domainIndex, plainTripTweet) + } + }.getOrElse(Seq.empty) + + scopedStats.counter(tripDomain.sourceId).incr(tripTweetsSeq.size) + tripTweetsSeq + } + }).map(_.flatten).map { tweetsWithIndicesSeq: Seq[(Int, Int, t.TripTweet)] => + // RoundRobin by domain: sort by domain first, then tweetIndex, + // so each domain (sourceId + others) can have equal weights being selected + tweetsWithIndicesSeq + .sortBy(t => (t._1, t._2)).map(_._3).distinct.take(maxPopTopicCandidates) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/TripStratoTopicQuery.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/TripStratoTopicQuery.scala new file mode 100644 index 000000000..72f77dab4 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets/TripStratoTopicQuery.scala @@ -0,0 +1,8 @@ +package com.twitter.tweet_mixer.candidate_source.popular_topic_tweets + +import com.twitter.trends.trip_v1.trip_tweets.{thriftscala => t} + +case class TripStratoTopicQuery( + domains: Seq[t.TripDomain], + maxCandidatesPerSource: Int, + maxPopTopicCandidates: Int) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/BUILD.bazel new file mode 100644 index 000000000..97fe5a951 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/BUILD.bazel @@ -0,0 +1,10 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/QigServiceBatchTweetCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/QigServiceBatchTweetCandidateSource.scala new file mode 100644 index 000000000..fb693c0a6 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/QigServiceBatchTweetCandidateSource.scala @@ -0,0 +1,57 @@ +package com.twitter.tweet_mixer.candidate_source.qig_service + +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.search.query_interaction_graph.service.{thriftscala => t} +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Calls QIG Service's getBatchCandidates endpoint. + * + * Since Candidate Sources need to return a flattened list of candidates, we wrap the TweetCandidate + * into a case class containing the relative rank of the Tweet and the rank of the request in the + * batch. + */ +@Singleton +class QigServiceBatchTweetCandidateSource @Inject() ( + qigService: t.QigService.MethodPerEndpoint) + extends CandidateSource[Seq[t.QigRequest], QigTweetCandidate] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "QigServiceBatchTweets") + + override def apply(requests: Seq[t.QigRequest]): Stitch[Seq[QigTweetCandidate]] = { + Stitch + .callFuture(qigService.getBatchCandidates(requests)) + .map { responses => + if (responses.size == requests.size) { + responses.zip(requests).map { + case (response, request) => + response.tweetCandidates match { + case Some(tweetCandidates) => + tweetCandidates.map { tweetCandidate => + QigTweetCandidate(tweetCandidate, request.query.getOrElse("")) + } + case _ => Seq.empty + } + } + } else { + throw PipelineFailure( + IllegalStateFailure, + s"QigService Batch endpoint returned ${responses.size} results but was expecting ${requests.size}" + ) + } + } + // round robbin flatten + .map { resultLists => + resultLists + .flatMap(list => list.zipWithIndex) + .sortBy(_._2) + .map(_._1) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/QigTweetCandidate.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/QigTweetCandidate.scala new file mode 100644 index 000000000..c64bc16e9 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service/QigTweetCandidate.scala @@ -0,0 +1,7 @@ +package com.twitter.tweet_mixer.candidate_source.qig_service + +import com.twitter.search.query_interaction_graph.service.{thriftscala => t} + +case class QigTweetCandidate( + tweetCandidate: t.QigTweetCandidate, + query: String) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/BUILD.bazel new file mode 100644 index 000000000..3ca5fa8b7 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/BUILD.bazel @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline", + "relevance-platform/src/main/scala/com/twitter/relevance_platform/simclustersann/multicluster", + "servo/repo", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "simclusters-ann/thrift/src/main/thrift:thrift-scala", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + "strato/config/columns/content_understanding:content_understanding-strato-client" + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SANNQuery.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SANNQuery.scala new file mode 100644 index 000000000..9786bd7a3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SANNQuery.scala @@ -0,0 +1,9 @@ +package com.twitter.tweet_mixer.candidate_source.simclusters_ann + +import com.twitter.simclustersann.thriftscala.{Query => SimClustersANNQuery} + +case class SANNQuery( + queries: Seq[SimClustersANNQuery], + maxCandidates: Int, + minScore: Double, + enableCache: Boolean) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimClustersAnnCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimClustersAnnCandidateSource.scala new file mode 100644 index 000000000..987c5a359 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimClustersAnnCandidateSource.scala @@ -0,0 +1,111 @@ +package com.twitter.tweet_mixer.candidate_source.simclusters_ann + +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandidateResult +import com.twitter.relevance_platform.simclustersann.multicluster.ServiceNameMapper +import com.twitter.servo.util.Transformer +import com.twitter.simclusters_v2.thriftscala.InternalId +import com.twitter.simclustersann.thriftscala.SimClustersANNService +import com.twitter.simclustersann.thriftscala.{Query => SimClustersANNQuery} +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SimClustersAnnCandidateSource @Inject() ( + simClustersANNServiceNameToClientMapper: Map[String, SimClustersANNService.MethodPerEndpoint], + memcacheClient: MemcacheStitchClient) + extends MemcachedCandidateSource[ + SANNQuery, + SimClustersANNQuery, + (Long, Double), + TweetMixerCandidate + ] { + + private def getSimClustersANNService( + query: SimClustersANNQuery + ): Option[SimClustersANNService.MethodPerEndpoint] = { + ServiceNameMapper + .getServiceName(query.sourceEmbeddingId.modelVersion, query.config.candidateEmbeddingType) + .flatMap { serviceName => + simClustersANNServiceNameToClientMapper.get(serviceName) + } + } + + private def getSeedId( + query: SimClustersANNQuery + ): Long = { + query.sourceEmbeddingId.internalId match { + case InternalId.UserId(userId) => userId + case InternalId.TweetId(tweetId) => tweetId + case _ => + throw PipelineFailure(UnexpectedCandidateResult, "Internal Id not of the supported types") + } + } + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "SimclustersAnnCandidateSource" + ) + + override val TTL = Utils.randomizedTTL(600) //10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: SimClustersANNQuery): String = "SANN:" + key.toString + + val valueTransformer: Transformer[Seq[(Long, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + + override def enableCache(request: SANNQuery): Boolean = request.enableCache + + override def getKeys(request: SANNQuery): Stitch[Seq[SimClustersANNQuery]] = + Stitch.value(request.queries) + + override def getCandidatesFromStore( + key: SimClustersANNQuery + ): Stitch[Seq[(Long, Double)]] = { + getSimClustersANNService(key) match { + // Find the service + case Some(simClustersANNService) => + val tweetCandidatesFuture = simClustersANNService.getTweetCandidates(key) + Stitch + .callFuture(tweetCandidatesFuture) + .map { + _.map { candidate => + (candidate.tweetId, candidate.score) + } + } + //Throw error when service is misconfigured + case None => + val errorMessage = + s"No SANN Cluster configured to serve this query, check CandidateEmbeddingType and ModelVersion: $key" + throw PipelineFailure(BadRequest, errorMessage) + } + } + + override def postProcess( + request: SANNQuery, + keys: Seq[SimClustersANNQuery], + resultsSeq: Seq[Seq[(Long, Double)]] + ): Seq[TweetMixerCandidate] = { + val maxCandidates = request.maxCandidates + val minScore = request.minScore + val filteredResults = keys.zip(resultsSeq).map { + case (key, results) => + val seedId = getSeedId(key) + results + .collect { + case (id, score) if score > minScore => TweetMixerCandidate(id, score, seedId) + }.take(maxCandidates) + } + TweetMixerCandidate.interleave(filteredResults) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterColdPostsCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterColdPostsCandidateSource.scala new file mode 100644 index 000000000..4c41564ed --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterColdPostsCandidateSource.scala @@ -0,0 +1,58 @@ +package com.twitter.tweet_mixer.candidate_source.simclusters_ann + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.content_understanding.UserInterestedSimclusterColdPostsClientColumn +import com.twitter.product_mixer.core.util.OffloadFuturePools +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Random +import scala.math.abs + +@Singleton +class SimclusterColdPostsCandidateSource @Inject() ( + simclusterColdPostsStratoColumn: UserInterestedSimclusterColdPostsClientColumn, + inputStatsReceiver: StatsReceiver) + extends CandidateSource[SimclusterColdPostsQuery, Long] { + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("SimclusterColdPosts") + + private val scopedStats = inputStatsReceiver.scope(getClass.getSimpleName) + private val candidatesStat = scopedStats.counter("candidatesSize") + + override def apply(request: SimclusterColdPostsQuery): Stitch[Seq[Long]] = { + getSimclusterPosts( + userId = request.userId, + postsPerSimcluster = request.postsPerSimcluster, + maxCandidates = request.maxCandidates + ) + } + private val fetcher = simclusterColdPostsStratoColumn.fetcher + + private def getSimclusterPosts( + userId: Long, + postsPerSimcluster: Int, + maxCandidates: Int + ): Stitch[Seq[Long]] = { + OffloadFuturePools.offloadStitch { + fetcher + .fetch( + UserInterestedSimclusterColdPostsClientColumn.Key( + userId = userId, + postsPerSimcluster = postsPerSimcluster + ) + ).map { result => + result.v match { + case Some(posts) => + val candidates: Seq[Long] = posts.map(v => abs(v._2)) + candidatesStat.incr(candidates.size) + Random.shuffle(candidates).take(maxCandidates) + case None => Seq.empty + } + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterColdPostsQuery.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterColdPostsQuery.scala new file mode 100644 index 000000000..c459efc5b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterColdPostsQuery.scala @@ -0,0 +1,7 @@ +package com.twitter.tweet_mixer.candidate_source.simclusters_ann + +case class SimclusterColdPostsQuery( + userId: Long, + postsPerSimcluster: Int, + maxCandidates: Int +) \ No newline at end of file diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterPromotedCreatorAnnCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterPromotedCreatorAnnCandidateSource.scala new file mode 100644 index 000000000..3b2bce493 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann/SimclusterPromotedCreatorAnnCandidateSource.scala @@ -0,0 +1,21 @@ +package com.twitter.tweet_mixer.candidate_source.simclusters_ann + +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.simclustersann.thriftscala.SimClustersANNService +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SimclusterPromotedCreatorAnnCandidateSource @Inject() ( + simClustersANNServiceNameToClientMapper: Map[String, SimClustersANNService.MethodPerEndpoint], + memcacheClient: MemcacheStitchClient) + extends SimClustersAnnCandidateSource( + simClustersANNServiceNameToClientMapper, + memcacheClient + ) { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( + "SimclusterPromotedCreatorAnnCandidateSource" + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/BUILD.bazel new file mode 100644 index 000000000..248fe349a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/BUILD.bazel @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "servo/repo", + "strato/config/columns/searchai/vectordb:vectordb-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "vecdb/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/TextEmbeddingCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/TextEmbeddingCandidateSource.scala new file mode 100644 index 000000000..264e81372 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/TextEmbeddingCandidateSource.scala @@ -0,0 +1,40 @@ +package com.twitter.tweet_mixer.candidate_source.text_embedding_ann + +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import com.twitter.strato.catalog.Scan.Slice +import com.twitter.strato.generated.client.searchai.vectordb.RealtimeTopV8ClientColumn +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TextEmbeddingCandidateSource @Inject() ( + column: RealtimeTopV8ClientColumn) + extends CandidateSource[Seq[TextEmbeddingQuery], TweetMixerCandidate] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("TextEmbedding") + + private val scanner = column.scanner + + override def apply(request: Seq[TextEmbeddingQuery]): Stitch[Seq[TweetMixerCandidate]] = { + + val stitchResponse = request.map { query => + val scan = Slice[Long](from = None, to = None, limit = Some(query.limit)) + val view = RealtimeTopV8ClientColumn.View( + vector = query.vector, + filter = None, + params = None + ) + scanner.scan(scan, view).map { + _.map { scoredPoint => + val value = scoredPoint._2 + TweetMixerCandidate(tweetId = value.id, score = value.score, seedId = -1L) + } + } + } + + Stitch.collect(stitchResponse).map(_.flatten) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/TextEmbeddingQuery.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/TextEmbeddingQuery.scala new file mode 100644 index 000000000..4854e8cdb --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann/TextEmbeddingQuery.scala @@ -0,0 +1,5 @@ +package com.twitter.tweet_mixer.candidate_source.text_embedding_ann + +case class TextEmbeddingQuery( + vector: Seq[Double], + limit: Int) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/BUILD.bazel new file mode 100644 index 000000000..09b9b1464 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/BUILD.bazel @@ -0,0 +1,16 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/com/twitter/storehaus:core", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "servo/repo", + "src/thrift/com/twitter/topic_recos:topic_recos-thrift-scala", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/CertoTopicTweetsCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/CertoTopicTweetsCandidateSource.scala new file mode 100644 index 000000000..00d7cf6a1 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/CertoTopicTweetsCandidateSource.scala @@ -0,0 +1,90 @@ +package com.twitter.tweet_mixer.candidate_source.topic_tweets + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.topic_recos.thriftscala.TweetWithScores +import com.twitter.simclusters_v2.thriftscala.TopicId +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +case class CertoTopicTweetsQuery( + topicIds: Seq[TopicId], + maxCandidatesPerTopic: Int, + maxCandidates: Int, + minCertoScore: Double, + minFavCount: Int) + +case class CertoSingleTopicTweetsQuery( + topicId: TopicId, + maxCandidatesPerTopic: Int, + minCertoScore: Double, + minFavCount: Int) + +@Singleton +class CertoTopicTweetsCandidateSource @Inject() ( + @Named(ModuleNames.CertoStratoTopicTweetsStoreName) + certoStratoTopicTweetsStore: ReadableStore[TopicId, Seq[TweetWithScores]], + inputStatsReceiver: StatsReceiver) + extends CandidateSource[CertoTopicTweetsQuery, TweetMixerCandidate] { + + private val scopedStats = inputStatsReceiver.scope(getClass.getSimpleName) + private val emptyTopicsCounter = scopedStats.counter("emptyTopics") + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("CertoTopicTweets") + + override def apply(request: CertoTopicTweetsQuery): Stitch[Seq[TweetMixerCandidate]] = { + val singleTopicIdQueries: Seq[CertoSingleTopicTweetsQuery] = request.topicIds.map { topicId => + CertoSingleTopicTweetsQuery( + topicId = topicId, + minCertoScore = request.minCertoScore, + maxCandidatesPerTopic = request.maxCandidatesPerTopic, + minFavCount = request.minFavCount + ) + } + + if (singleTopicIdQueries.isEmpty) { + emptyTopicsCounter.incr() + Stitch.value(Seq.empty) + } else { + val resultsStitch: Stitch[Seq[Seq[TweetMixerCandidate]]] = + Stitch.traverse(singleTopicIdQueries)(fetchFromCertoStore) + + // this will interleave the candidates by topic for diversity + resultsStitch.map { results: Seq[Seq[TweetMixerCandidate]] => + TweetMixerCandidate.interleave(results) + } + } + } + + private def fetchFromCertoStore( + query: CertoSingleTopicTweetsQuery + ): Stitch[Seq[TweetMixerCandidate]] = { + val tweetsWithScoresF = certoStratoTopicTweetsStore.get(query.topicId) + + Stitch.callFuture(tweetsWithScoresF).map { tweetsWithScoresOpt => + { + val tweetsWithScores: Seq[TweetWithScores] = tweetsWithScoresOpt + .getOrElse(Seq.empty) + .filter(_.scores.followerL2NormalizedCosineSimilarity8HrHalfLife >= query.minCertoScore) + .filter(_.scores.favCount.exists(_ >= query.minFavCount)) + .sortBy(-_.scores.favCount.getOrElse(0L)) + .take(query.maxCandidatesPerTopic) + + tweetsWithScores.map { tweetWithScore => + TweetMixerCandidate( + tweetId = tweetWithScore.tweetId, + score = tweetWithScore.scores.favCount.getOrElse(0L).toDouble, + seedId = query.topicId.entityId + ) + } + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/SkitTopicTweetsCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/SkitTopicTweetsCandidateSource.scala new file mode 100644 index 000000000..3e87cdfb3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets/SkitTopicTweetsCandidateSource.scala @@ -0,0 +1,78 @@ +package com.twitter.tweet_mixer.candidate_source.topic_tweets + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.topic_recos.thriftscala.TopicTweet +import com.twitter.topic_recos.thriftscala.TopicTweetPartitionFlatKey +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +case class SkitTopicTweetsQuery( + topicKeys: Seq[SkitTimedTopicKeys], + maxCandidatesPerTopic: Int, + maxCandidates: Int, + minScore: Double, + minFavCount: Int) + +case class SkitTimedTopicKeys(keys: Seq[TopicTweetPartitionFlatKey], topicId: Long) + +@Singleton +class SkitTopicTweetsCandidateSource @Inject() ( + @Named(ModuleNames.SkitStratoTopicTweetsStoreName) + skitStratoTopicTweetsStore: ReadableStore[TopicTweetPartitionFlatKey, Seq[TopicTweet]], + inputStatsReceiver: StatsReceiver) + extends CandidateSource[SkitTopicTweetsQuery, TweetMixerCandidate] { + + private val scopedStats = inputStatsReceiver.scope(getClass.getSimpleName) + private val emptyTopicsCounter = scopedStats.counter("emptyTopics") + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("SkitTopicTweets") + + override def apply(request: SkitTopicTweetsQuery): Stitch[Seq[TweetMixerCandidate]] = { + + if (request.topicKeys.isEmpty) { + emptyTopicsCounter.incr() + Stitch.value(Seq.empty) + } else { + val resultsStitch: Stitch[Seq[Seq[TweetMixerCandidate]]] = + Stitch.traverse(request.topicKeys) { topicKey: SkitTimedTopicKeys => + Stitch + .traverse(topicKey.keys) { skitKey => + Stitch.callFuture(skitStratoTopicTweetsStore.get(skitKey)) + }.map { topicTweets => + val flattenedSkitTopicTweets: Seq[TopicTweet] = + topicTweets + .collect { + case Some(topicTweets: Seq[TopicTweet]) => + topicTweets.filter { topicTweet => + topicTweet.scores.cosineSimilarity.exists(_ > request.minScore) && + topicTweet.scores.favCount.exists(_ > request.minFavCount.toLong) + } + }.flatten + .sortBy(-_.scores.favCount.getOrElse(0L)) + .take(request.maxCandidatesPerTopic) + + flattenedSkitTopicTweets.map { skitTopicTweet => + TweetMixerCandidate( + skitTopicTweet.tweetId, + score = skitTopicTweet.scores.favCount.getOrElse(0L).toDouble, + seedId = topicKey.topicId + ) + } + } + } + + // Interleave the candidates by topic for diversity + resultsStitch.map { results: Seq[Seq[TweetMixerCandidate]] => + TweetMixerCandidate.interleave(results) + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/BUILD.bazel new file mode 100644 index 000000000..acdec4139 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/BUILD.bazel @@ -0,0 +1,16 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "servo/repo/src/main/scala", + "strato/config/columns/recommendations/simclusters_v2:simclusters_v2-strato-client", + "strato/config/columns/trendsai/media:media-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/TrendsCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/TrendsCandidateSource.scala new file mode 100644 index 000000000..bb6fbfb90 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/TrendsCandidateSource.scala @@ -0,0 +1,63 @@ +package com.twitter.tweet_mixer.candidate_source.trends + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ExpiringLruInProcessCache +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Client +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.recommendations.simclusters_v2.TopPostsPerCountryClientColumn +import com.twitter.tweet_mixer.candidate_source.trends.TrendsCandidateSource.cache +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.util.Random + +object TrendsCandidateSource { + private val BaseTTL = 5 + private val TTL = (BaseTTL + Random.nextInt(5)).minutes + + val cache: InProcessCache[String, Seq[TweetMixerCandidate]] = + new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 500) +} + +@Singleton +class TrendsCandidateSource @Inject() ( + @Named("StratoClientWithModerateTimeout") stratoClient: Client) + extends CandidateSource[ + TopPostsPerCountryClientColumn.Key, + TweetMixerCandidate + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("Trends") + + private val fetcher: Fetcher[ + TopPostsPerCountryClientColumn.Key, + TopPostsPerCountryClientColumn.View, + TopPostsPerCountryClientColumn.Value + ] = new TopPostsPerCountryClientColumn(stratoClient).fetcher + + private val DefaultScore = 0 + + override def apply( + key: String + ): Stitch[Seq[TweetMixerCandidate]] = OffloadFuturePools.offloadStitch { + cache.get(key).map(Stitch.value(_)).getOrElse { + fetcher.fetch(key).map { result => + val candidates = result.v.getOrElse(Seq.empty).map { post => + TweetMixerCandidate( + tweetId = post.tweetId, + score = DefaultScore, + seedId = post.clusterId + ) + } + cache.set(key, candidates) + candidates + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/TrendsVideoCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/TrendsVideoCandidateSource.scala new file mode 100644 index 000000000..84a124d4a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/trends/TrendsVideoCandidateSource.scala @@ -0,0 +1,62 @@ +package com.twitter.tweet_mixer.candidate_source.trends + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.servo.cache.ExpiringLruInProcessCache +import com.twitter.servo.cache.InProcessCache +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Client +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.trendsai.media.TopCountryVideosClientColumn +import com.twitter.tweet_mixer.candidate_source.trends.TrendsVideoCandidateSource.cache +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate + +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.util.Random + +object TrendsVideoCandidateSource { + private val BaseTTL = 5 + private val TTL = (BaseTTL + Random.nextInt(5)).minutes + + val cache: InProcessCache[String, Seq[TweetMixerCandidate]] = + new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 500) +} + +@Singleton +class TrendsVideoCandidateSource @Inject() ( + @Named("StratoClientWithModerateTimeout") stratoClient: Client) + extends CandidateSource[ + TopCountryVideosClientColumn.Key, + TweetMixerCandidate + ] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("TrendsVideo") + + private val fetcher: Fetcher[ + TopCountryVideosClientColumn.Key, + TopCountryVideosClientColumn.View, + TopCountryVideosClientColumn.Value + ] = new TopCountryVideosClientColumn(stratoClient).fetcher + + override def apply( + key: String + ): Stitch[Seq[TweetMixerCandidate]] = OffloadFuturePools.offloadStitch { + cache.get(key).map(Stitch.value(_)).getOrElse { + fetcher.fetch(key).map { result => + val candidates = result.v.getOrElse(Seq.empty).map { post => + TweetMixerCandidate( + tweetId = post.postId, + score = post.score, + seedId = post.trendId + ) + } + cache.set(key, candidates) + candidates + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/BUILD.bazel new file mode 100644 index 000000000..567fa9421 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/BUILD.bazel @@ -0,0 +1,21 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/com/twitter/storehaus:core", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "servo/repo", + "simclusters-ann/thrift/src/main/thrift:thrift-scala", + "src/scala/com/twitter/simclusters_v2/common", + "timelines/src/main/scala/com/twitter/timelines/clients/ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/cached_candidate_source", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + "vecdb/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINANNCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINANNCandidateSource.scala new file mode 100644 index 000000000..ab18c63cb --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINANNCandidateSource.scala @@ -0,0 +1,124 @@ +package com.twitter.tweet_mixer.candidate_source.twhin_ann + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.servo.util.Transformer +import com.twitter.simclusters_v2.common.TweetId +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.clients.ann.ANNQdrantGRPCClient +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.utils.BucketSnowflakeIdAgeStats +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class TwHINANNCandidateSource @Inject() ( + @Named(ModuleNames.TwHINTweetEmbeddingStratoStore) + tweetEmbeddingStore: ReadableStore[TweetId, TwhinTweetEmbedding], + @Named(ModuleNames.TwHINANNServiceClient) + annClient: ANNQdrantGRPCClient, + memcacheClient: MemcacheStitchClient, + statsReceiver: StatsReceiver) + extends MemcachedCandidateSource[ + Seq[TweetId], + TweetId, + (TweetId, Double), + TweetMixerCandidate + ] { + + import TwHINANNCandidateSource._ + + private val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetScoreStats = scopedStats.stat("tweetScore") + private val tweetSizePerSignalStats = scopedStats.stat("tweetSizePerSignal") + private val tweetAgeStats = + BucketSnowflakeIdAgeStats[TweetMixerCandidate]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _.tweetId)(scopedStats.scope("tweetAge")) + private val missingQueryEmbeddingCounter = scopedStats.counter("missingQueryEmbedding") + private val emptyResultCounter = scopedStats.counter("emptyResult") + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("TwHINANNCandidateSource") + + override val TTL: Int = Utils.randomizedTTL(600) //10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: TweetId): String = { + "TwHIN:" + key.toString + } + + val valueTransformer: Transformer[Seq[(Long, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + override def getKeys(request: Seq[TweetId]): Stitch[Seq[TweetId]] = Stitch.value(request) + + override def getCandidatesFromStore( + key: TweetId + ): Stitch[Seq[(TweetId, Double)]] = { + val futureResult = for { + tweetEmbedding <- tweetEmbeddingStore.get(key).map(_.map(toVector)) + response <- tweetEmbedding match { + case Some(embedding) => + annClient.searchANN( + embedding = embedding, + topK = 100, + collectionName = "twhin-prod", + timeout = 20.millis) + case None => + missingQueryEmbeddingCounter.incr() + Future.Nil + } + } yield { + val result = response.map { + case (tweetId, score) => + (tweetId, score.toDouble) + } + result + } + Stitch.callFuture(futureResult) + } + + override def postProcess( + request: Seq[TweetId], + keys: Seq[TweetId], + resultsSeq: Seq[Seq[(TweetId, Double)]] + ): Seq[TweetMixerCandidate] = { + val twHINCandidates = keys.zip(resultsSeq).map { + case (key, results) => + results + .map { + case (id, score) => + tweetScoreStats.add(score.toFloat * 1000) + TweetMixerCandidate(id, score, key) + } + } + twHINCandidates.foreach { seq => + val seqSize = seq.size + if (seqSize == 0) { + emptyResultCounter.incr() + } + tweetSizePerSignalStats.add(seqSize) + } + val result = TweetMixerCandidate.interleave(twHINCandidates) + tweetAgeStats.count(result) + result + } +} + +object TwHINANNCandidateSource { + def toVector(embedding: TwhinTweetEmbedding): Seq[Float] = { + embedding.embedding.map(_.toFloat) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINRebuildANNCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINRebuildANNCandidateSource.scala new file mode 100644 index 000000000..e16699439 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINRebuildANNCandidateSource.scala @@ -0,0 +1,134 @@ +package com.twitter.tweet_mixer.candidate_source.twhin_ann + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.io.Buf +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.servo.util.Transformer +import com.twitter.simclusters_v2.common.TweetId +import com.twitter.simclusters_v2.common.VersionId +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.tweet_mixer.candidate_source.cached_candidate_source.MemcachedCandidateSource +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import com.twitter.tweet_mixer.utils.BucketSnowflakeIdAgeStats +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import com.twitter.util.Future +import com.twitter.vecdb.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +class TwHINRebuildANNCandidateSource @Inject() ( + @Named(ModuleNames.TwHINRebuildTweetEmbeddingStratoStore) + tweetEmbeddingStore: ReadableStore[(TweetId, VersionId), TwhinTweetEmbedding], + @Named(ModuleNames.VecDBAnnServiceClient) + annClient: t.VecDB.MethodPerEndpoint, + memcacheClient: MemcacheStitchClient, + statsReceiver: StatsReceiver) + extends MemcachedCandidateSource[ + Seq[TwHINRebuildANNKey], + TwHINRebuildANNKey, + (TweetId, Double), + TweetMixerCandidate + ] { + + import TwHINRebuildANNCandidateSource._ + + private val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val tweetScoreStats = scopedStats.stat("tweetScore") + private val tweetSizePerSignalStats = scopedStats.stat("tweetSizePerSignal") + private val tweetAgeStats = + BucketSnowflakeIdAgeStats[TweetMixerCandidate]( + BucketSnowflakeIdAgeStats.MillisecondsPerHour, + _.tweetId)(scopedStats.scope("tweetAge")) + private val foundQueryEmbeddingCounter = scopedStats.counter("foundQueryEmbedding") + private val missingQueryEmbeddingCounter = scopedStats.counter("missingQueryEmbedding") + private val successVecDBRequestsCounter = scopedStats.counter("successVecDBRequests") + private val failedVecDBRequestsCounter = scopedStats.counter("failedVecDBRequests") + + override val identifier: CandidateSourceIdentifier = + CandidateSourceIdentifier("TwHINRebuildANN") + + override val TTL: Int = Utils.randomizedTTL(600) //10 minutes + + override val memcache: MemcacheStitchClient = memcacheClient + + override def keyTransformer(key: TwHINRebuildANNKey): String = { + "TwHINRebuild:" + key.toString + } + + val valueTransformer: Transformer[Seq[(Long, Double)], Buf] = + Transformers.longDoubleSeqBufTransformer + override def getKeys(request: Seq[TwHINRebuildANNKey]): Stitch[Seq[TwHINRebuildANNKey]] = + Stitch.value(request) + + override def getCandidatesFromStore( + key: TwHINRebuildANNKey + ): Stitch[Seq[(TweetId, Double)]] = { + val futureResult = + tweetEmbeddingStore + .get((key.id, key.versionId)).map(_.map(toVector)).flatMap { + case Some(embedding) => + foundQueryEmbeddingCounter.incr() + annClient + .search( + dataset = key.dataset, + vector = embedding, + params = Some(t.SearchParams(limit = Some(key.maxCandidates))) + ).map { response: t.SearchResponse => + response.points match { + case points: Seq[t.ScoredPoint] => + successVecDBRequestsCounter.incr() + points.map { point => + (point.id, point.score) + } + case _ => + failedVecDBRequestsCounter.incr() + Seq.empty + } + } + case None => + missingQueryEmbeddingCounter.incr() + Future.Nil + }.rescue({ + case _: Exception => + failedVecDBRequestsCounter.incr() + Future.Nil + }) + Stitch.callFuture(futureResult) + } + + override def postProcess( + request: Seq[TwHINRebuildANNKey], + keys: Seq[TwHINRebuildANNKey], + resultsSeq: Seq[Seq[(TweetId, Double)]] + ): Seq[TweetMixerCandidate] = { + val twHINCandidates = keys.zip(resultsSeq).map { + case (key, results) => + results + .map { + case (id, score) => + tweetScoreStats.add(score.toFloat * 1000) + TweetMixerCandidate(id, score, key.id) + } + } + twHINCandidates.foreach { seq => + val seqSize = seq.size + tweetSizePerSignalStats.add(seqSize) + } + val result = TweetMixerCandidate.interleave(twHINCandidates) + tweetAgeStats.count(result) + result + } +} + +object TwHINRebuildANNCandidateSource { + def toVector(embedding: TwhinTweetEmbedding): Seq[Double] = { + embedding.embedding + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINRebuildANNKey.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINRebuildANNKey.scala new file mode 100644 index 000000000..30930b225 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann/TwHINRebuildANNKey.scala @@ -0,0 +1,10 @@ +package com.twitter.tweet_mixer.candidate_source.twhin_ann + +import com.twitter.simclusters_v2.common.TweetId +import com.twitter.simclusters_v2.common.VersionId + +case class TwHINRebuildANNKey( + id: TweetId, + dataset: String, + versionId: VersionId, + maxCandidates: Int) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/user_location/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/user_location/BUILD.bazel new file mode 100644 index 000000000..d7b468eb9 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/user_location/BUILD.bazel @@ -0,0 +1,13 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "strato/config/columns/timelines/local:local-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/user_location/UserLocationCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/user_location/UserLocationCandidateSource.scala new file mode 100644 index 000000000..74666ed3d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/user_location/UserLocationCandidateSource.scala @@ -0,0 +1,45 @@ +package com.twitter.tweet_mixer.candidate_source.user_location + +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.timelines.local.FetchLocalPostsClientColumn +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserLocationCandidateSource @Inject() ( + fetchLocalPostsClientColumn: FetchLocalPostsClientColumn) + extends StratoKeyViewFetcherSource[ + FetchLocalPostsClientColumn.Key, + FetchLocalPostsClientColumn.View, + FetchLocalPostsClientColumn.Value, + TweetMixerCandidate + ] { + + override val fetcher: Fetcher[ + FetchLocalPostsClientColumn.Key, + FetchLocalPostsClientColumn.View, + FetchLocalPostsClientColumn.Value + ] = fetchLocalPostsClientColumn.fetcher + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("UserLocation") + private val DefaultSeedId = 0 + private val MaxSearchResults = 1000 + + override protected def stratoResultTransformer( + stratoKey: FetchLocalPostsClientColumn.Key, + stratoResult: FetchLocalPostsClientColumn.Value + ): Seq[TweetMixerCandidate] = { + stratoResult.search + .take(MaxSearchResults) + .map { post => + TweetMixerCandidate( + tweetId = post.id, + score = post.score, + seedId = post.cityId.getOrElse(DefaultSeedId) + ) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/uss_service/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/uss_service/BUILD.bazel new file mode 100644 index 000000000..a1495056a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/uss_service/BUILD.bazel @@ -0,0 +1,12 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "strato/config/columns/recommendations/user-signal-service:user-signal-service-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/uss_service/USSSignalCandidateSource.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/uss_service/USSSignalCandidateSource.scala new file mode 100644 index 000000000..e72fb5d5c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/uss_service/USSSignalCandidateSource.scala @@ -0,0 +1,34 @@ +package com.twitter.tweet_mixer.candidate_source.uss_service + +import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyFetcherSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.recommendations.user_signal_service.SignalsClientColumn +import com.twitter.usersignalservice.thriftscala.SignalType +import com.twitter.usersignalservice.thriftscala.{Signal => UssSignal} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class USSSignalCandidateSource @Inject() ( + signalsClientColumn: SignalsClientColumn) + extends StratoKeyFetcherSource[ + SignalsClientColumn.Key, + SignalsClientColumn.Value, + (SignalType, Seq[UssSignal]) + ] { + + override val fetcher: Fetcher[SignalsClientColumn.Key, Unit, SignalsClientColumn.Value] = + signalsClientColumn.fetcher + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("USSSignal") + + override protected def stratoResultTransformer( + stratoResult: signalsClientColumn.Value + ): Seq[(SignalType, Seq[UssSignal])] = { + stratoResult.signalResponse.toSeq.map { + case (signalType, signals) => + (signalType, signals) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/BUILD.bazel new file mode 100644 index 000000000..785f18c43 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/BUILD.bazel @@ -0,0 +1,9 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "simclusters-ann/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/SimClustersANNConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/SimClustersANNConfig.scala new file mode 100644 index 000000000..0241d51dc --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/SimClustersANNConfig.scala @@ -0,0 +1,199 @@ +package com.twitter.tweet_mixer.config + +import com.twitter.conversions.DurationOps._ +import com.twitter.simclusters_v2.thriftscala.EmbeddingType +import com.twitter.simclusters_v2.thriftscala.EmbeddingType.LogFavBasedVideoTweet +import com.twitter.simclusters_v2.thriftscala.InternalId +import com.twitter.simclusters_v2.thriftscala.ModelVersion +import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId +import com.twitter.simclustersann.thriftscala.ScoringAlgorithm +import com.twitter.simclustersann.thriftscala.{SimClustersANNConfig => ThriftSimClustersANNConfig} +import com.twitter.simclustersann.thriftscala.{Query => SimClustersANNQuery} +import com.twitter.util.Duration + +case class SimClustersANNConfig( + maxNumResults: Int, + minScore: Double, + candidateEmbeddingType: EmbeddingType, + maxTopTweetsPerCluster: Int, + maxScanClusters: Int, + maxTweetCandidateAge: Duration, + minTweetCandidateAge: Duration, + annAlgorithm: ScoringAlgorithm, + engagementThreshold: Option[Double] = None, + isClusterDetailBasedFilteringEnabled: Option[Boolean] = None, + clusterDetailBasedThreshold: Option[Double] = None) { + val toSANNConfigThrift: ThriftSimClustersANNConfig = ThriftSimClustersANNConfig( + maxNumResults = maxNumResults, + minScore = minScore, + candidateEmbeddingType = candidateEmbeddingType, + maxTopTweetsPerCluster = maxTopTweetsPerCluster, + maxScanClusters = maxScanClusters, + maxTweetCandidateAgeHours = maxTweetCandidateAge.inHours, + minTweetCandidateAgeHours = minTweetCandidateAge.inHours, + annAlgorithm = annAlgorithm, + engagementThreshold = engagementThreshold, + isClusterDetailBasedFilteringEnabled = isClusterDetailBasedFilteringEnabled, + clusterDetailBasedThreshold = clusterDetailBasedThreshold + ) +} + +object SimClustersANNConfig { + + final val DefaultConfig = SimClustersANNConfig( + maxNumResults = 200, + minScore = 0.0, + candidateEmbeddingType = EmbeddingType.LogFavBasedTweet, + maxTopTweetsPerCluster = 800, + maxScanClusters = 50, + maxTweetCandidateAge = 24.hours, + minTweetCandidateAge = 0.hours, + annAlgorithm = ScoringAlgorithm.CosineSimilarity, + ) + + final val FavL2NormConfig = SimClustersANNConfig( + maxNumResults = 200, + minScore = 0.8, // FROM PROD CONFIG + candidateEmbeddingType = EmbeddingType.LogFavBasedTweet, + maxTopTweetsPerCluster = 800, + maxScanClusters = 50, + maxTweetCandidateAge = 48.hours, + minTweetCandidateAge = 0.hours, + annAlgorithm = ScoringAlgorithm.CosineSimilarityFavL2NormBased, + ) + + final val FavL2NormExplorationDefaultConfig = SimClustersANNConfig( + maxNumResults = 200, + minScore = 0.8, + candidateEmbeddingType = EmbeddingType.LogFavBasedTweet, + maxTopTweetsPerCluster = 800, + maxScanClusters = 50, + maxTweetCandidateAge = 48.hours, + minTweetCandidateAge = 0.hours, + annAlgorithm = ScoringAlgorithm.CosineSimilarityFavL2NormBasedWithExploration, + engagementThreshold = Some(1.0) + ) + + final val ClusterDetailBasedFilteringDefaultConfig = SimClustersANNConfig( + maxNumResults = 200, + minScore = 0.0, + candidateEmbeddingType = EmbeddingType.LogFavBasedTweet, + maxTopTweetsPerCluster = 800, + maxScanClusters = 50, + maxTweetCandidateAge = 24.hours, + minTweetCandidateAge = 0.hours, + annAlgorithm = ScoringAlgorithm.CosineSimilarity, + engagementThreshold = Some(1.0), + isClusterDetailBasedFilteringEnabled = Some(true), + clusterDetailBasedThreshold = Some(0.0) + ) + + final val ClusterDetailBasedFiltering0001Config = ClusterDetailBasedFilteringDefaultConfig.copy(clusterDetailBasedThreshold = Some(0.001)) + final val ClusterDetailBasedFiltering0005Config = ClusterDetailBasedFilteringDefaultConfig.copy(clusterDetailBasedThreshold = Some(0.005)) + final val ClusterDetailBasedFiltering001Config = ClusterDetailBasedFilteringDefaultConfig.copy(clusterDetailBasedThreshold = Some(0.010)) + final val ClusterDetailBasedFiltering0015Config = ClusterDetailBasedFilteringDefaultConfig.copy(clusterDetailBasedThreshold = Some(0.015)) + + + final val FavL2NormExploration11Config = FavL2NormExplorationDefaultConfig.copy(engagementThreshold = Some(1.1)) + final val FavL2NormExploration12Config = FavL2NormExplorationDefaultConfig.copy(engagementThreshold = Some(1.2)) + final val FavL2NormExploration13Config = FavL2NormExplorationDefaultConfig.copy(engagementThreshold = Some(1.3)) + final val FavL2NormExploration14Config = FavL2NormExplorationDefaultConfig.copy(engagementThreshold = Some(1.4)) + + /* + SimClustersANNConfigId: String + Format: Prod - “EmbeddingType_ModelVersion_Default” + Format: Experiment - “EmbeddingType_ModelVersion_Date_Two-Digit-Serial-Number”. Date : YYYYMMDD + */ + + private val ExtendedAgeTweetConfig = DefaultConfig.copy(maxTweetCandidateAge = 48.hours) + + private val MaxExtendedAgeTweetConfig = DefaultConfig.copy(maxTweetCandidateAge = 72.hours) + + private val VideoTweetConfig = + DefaultConfig.copy( + candidateEmbeddingType = LogFavBasedVideoTweet, + maxTweetCandidateAge = 24.hours) + + private val ExtendedAgeVideoTweetConfig = VideoTweetConfig.copy(maxTweetCandidateAge = 48.hours) + + private val MaxAgeVideoTweetConfig = VideoTweetConfig.copy(maxTweetCandidateAge = 168.hours) + + private val MoreRelatedMaxAgeVideoTweetConfig = VideoTweetConfig.copy( + maxTweetCandidateAge = 168.hours, + maxScanClusters = 10, + annAlgorithm = ScoringAlgorithm.LogCosineSimilarity) + + + val SourceToTweetEmbeddingConfigMappings: Map[String, SimClustersANNConfig] = Map( + "FavBasedProducer_Model20m145k2020_Default" -> DefaultConfig, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default" -> DefaultConfig, + "UnfilteredUserInterestedIn_Model20m145k2020_Default" -> DefaultConfig, + "FollowBasedUserInterestedIn_Model20m145k2020_Default" -> DefaultConfig, + "FavBasedProducer_Model20m145k2020_20220617_06" -> ExtendedAgeTweetConfig, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220617_06" -> ExtendedAgeTweetConfig, + "UnfilteredUserInterestedIn_Model20m145k2020_20220617_06" -> ExtendedAgeTweetConfig, + "FollowBasedUserInterestedIn_Model20m145k2020_20220617_06" -> ExtendedAgeTweetConfig, + "FavBasedProducer_Model20m145k2020_MaxAge3Days" -> MaxExtendedAgeTweetConfig, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_MaxAge3Days" -> MaxExtendedAgeTweetConfig, + "UnfilteredUserInterestedIn_Model20m145k2020_MaxAge3Days" -> MaxExtendedAgeTweetConfig, + "FollowBasedUserInterestedIn_Model20m145k2020_MaxAge3Days" -> MaxExtendedAgeTweetConfig, + "FavBasedProducer_Model20m145k2020_Video" -> VideoTweetConfig, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_Video" -> VideoTweetConfig, + "UnfilteredUserInterestedIn_Model20m145k2020_Video" -> VideoTweetConfig, + "FollowBasedUserInterestedIn_Model20m145k2020_Video" -> VideoTweetConfig, + "FavBasedProducer_Model20m145k2020_20220617_06_Video" -> ExtendedAgeVideoTweetConfig, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220617_06_Video" -> ExtendedAgeVideoTweetConfig, + "UnfilteredUserInterestedIn_Model20m145k2020_20220617_06_Video" -> ExtendedAgeVideoTweetConfig, + "FollowBasedUserInterestedIn_Model20m145k2020_20220617_06_Video" -> ExtendedAgeVideoTweetConfig, + "FavBasedProducer_Model20m145k2020_7DayVideo" -> MaxAgeVideoTweetConfig, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_7DayVideo" -> MaxAgeVideoTweetConfig, + "UnfilteredUserInterestedIn_Model20m145k2020_7DayVideo" -> MaxAgeVideoTweetConfig, + "FollowBasedUserInterestedIn_Model20m145k2020_7DayVideo" -> MaxAgeVideoTweetConfig, + "FavBasedProducer_Model20m145k2020_MoreRelatedMaxAge" -> MoreRelatedMaxAgeVideoTweetConfig, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_MoreRelatedMaxAge" -> MoreRelatedMaxAgeVideoTweetConfig, + "UnfilteredUserInterestedIn_Model20m145k2020_MoreRelatedMaxAge" -> MoreRelatedMaxAgeVideoTweetConfig, + "FollowBasedUserInterestedIn_Model20m145k2020_MoreRelatedMaxAge" -> MoreRelatedMaxAgeVideoTweetConfig, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_FavL2Norm" -> FavL2NormConfig, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_FavL2NormExploration" -> FavL2NormExplorationDefaultConfig, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_FavL2NormExploration11" -> FavL2NormExploration11Config, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_FavL2NormExploration12" -> FavL2NormExploration12Config, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_FavL2NormExploration13" -> FavL2NormExploration13Config, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_FavL2NormExploration14" -> FavL2NormExploration14Config, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_ClusterDetailBasedFiltering" -> ClusterDetailBasedFilteringDefaultConfig, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_ClusterDetailBasedFiltering0001" -> ClusterDetailBasedFiltering0001Config, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_ClusterDetailBasedFiltering0005" -> ClusterDetailBasedFiltering0005Config, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_ClusterDetailBasedFiltering001" -> ClusterDetailBasedFiltering001Config, + "LogFavLongestL2EmbeddingTweet_Model20m145k2020_ClusterDetailBasedFiltering0015" -> ClusterDetailBasedFiltering0015Config, + ) + + def getConfig( + embeddingType: String, + modelVersion: String, + id: String + ): SimClustersANNConfig = { + val configName = embeddingType + "_" + modelVersion + "_" + id + SourceToTweetEmbeddingConfigMappings.get(configName) match { + case Some(config) => config + case None => throw new Exception(s"Incorrect config id passed in for SANN $configName") + } + } + + def getQuery( + internalId: InternalId, + embeddingType: EmbeddingType, + modelVersion: ModelVersion, + simClustersANNConfigId: String, + ): SimClustersANNQuery = { + + // SimClusters EmbeddingId and ANNConfig + val simClustersEmbeddingId = + SimClustersEmbeddingId(embeddingType, modelVersion, internalId) + val simClustersANNConfig = + getConfig(embeddingType.toString, modelVersion.toString, simClustersANNConfigId) + + SimClustersANNQuery( + sourceEmbeddingId = simClustersEmbeddingId, + config = simClustersANNConfig.toSANNConfigThrift + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/TimeoutConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/TimeoutConfig.scala new file mode 100644 index 000000000..92546edf7 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config/TimeoutConfig.scala @@ -0,0 +1,17 @@ +package com.twitter.tweet_mixer.config + +import com.twitter.util.Duration + +/** + * Specifies the timeout budgets of various components. We need these to limit how much time to + * spend on each step (e.x. candidate sources should not take more than 100ms). + */ +case class TimeoutConfig( + thriftAnnServiceClientTimeout: Duration, + thriftSANNServiceClientTimeout: Duration, + thriftTweetypieClientTimeout: Duration, + thriftUserTweetGraphClientTimeout: Duration, + thriftUserVideoGraphClientTimeout: Duration, + candidateSourceTimeout: Duration, + userStateStoreTimeout: Duration +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/controller/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/controller/BUILD.bazel new file mode 100644 index 000000000..7d1e5dd9d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/controller/BUILD.bazel @@ -0,0 +1,23 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/com/twitter/storehaus:memcache", + "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query", + "snowflake/src/main/scala/com/twitter/snowflake/id", + "src/scala/com/twitter/simclusters_v2/common", + "src/scala/com/twitter/storehaus_internal/manhattan", + "src/scala/com/twitter/storehaus_internal/memcache", + "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/imv_related_tweets/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + "twitter-context/src/main/scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/controller/TweetMixerThriftController.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/controller/TweetMixerThriftController.scala new file mode 100644 index 000000000..ffc543a68 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/controller/TweetMixerThriftController.scala @@ -0,0 +1,79 @@ +package com.twitter.tweet_mixer.controller + +import com.twitter.core_workflows.user_model.thriftscala.UserState +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finatra.thrift.Controller +import com.twitter.product_mixer.core.controllers.DebugTwitterContext +import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder +import com.twitter.product_mixer.core.service.debug_query.DebugQueryService +import com.twitter.simclusters_v2.common.UserId +import com.twitter.timelines.configapi.Params +import com.twitter.tweet_mixer.service.TweetMixerService +import com.twitter.tweet_mixer.marshaller.request.TweetMixerRequestUnmarshaller +import com.twitter.tweet_mixer.model.request.TweetMixerRequest +import com.twitter.tweet_mixer.{thriftscala => t} +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.tweet_mixer.product.imv_related_tweets.model.request.IMVRelatedTweetsProductContext +import com.twitter.util.Future +import javax.inject.Inject + +class TweetMixerThriftController @Inject() ( + tweetMixerRequestUnmarshaller: TweetMixerRequestUnmarshaller, + tweetMixerService: TweetMixerService, + debugQueryService: DebugQueryService, + userStateStore: ReadableStore[UserId, UserState], + statsReceiver: StatsReceiver, + paramsBuilder: ParamsBuilder) + extends Controller(t.TweetMixer) + with DebugTwitterContext { + + handle(t.TweetMixer.GetRecommendationResponse) { + args: t.TweetMixer.GetRecommendationResponse.Args => + val request = tweetMixerRequestUnmarshaller(args.request) + val paramsFuture = buildParams(request) + paramsFuture.flatMap { params => + Stitch.run( + tweetMixerService + .getTweetMixerRecommendationResponse[TweetMixerRequest](request, params)) + } + } + + // Handle debug requests from Turntable + handle(t.TweetMixer.ExecutePipeline) + .withService(debugQueryService(tweetMixerRequestUnmarshaller.apply)) + + private def buildParams(request: TweetMixerRequest): Future[Params] = { + val userStateOptFut = + userStateStore.get(request.getUserIdLoggedOutSupport).handle { case _ => None } + userStateOptFut.map { userStateOpt => + val userState = userStateOpt.getOrElse(UserState.EnumUnknownUserState(100)) + statsReceiver.scope("UserState").counter(userState.toString).incr() + + val userAgeOpt: Option[Int] = request.clientContext.userId.map { userId => + SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) + } + + val imvRequestType: String = request.productContext + .flatMap { + case context: IMVRelatedTweetsProductContext => + context.requestType.map(_.name.toLowerCase) + case _ => None + }.getOrElse("") + + val fsCustomMapInput: Map[String, Any] = + Map( + "user_state" -> userState.toString, + "account_age_in_days" -> userAgeOpt.getOrElse(Int.MaxValue), + "imv_request_type" -> imvRequestType) + + paramsBuilder.build( + clientContext = request.clientContext, + product = request.product, + featureOverrides = request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), + fsCustomMapInput = fsCustomMapInput + ) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/BUILD.bazel new file mode 100644 index 000000000..4114f8425 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/BUILD.bazel @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + ], + exports = [ + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/EntityTypes.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/EntityTypes.scala new file mode 100644 index 000000000..699fd280f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/EntityTypes.scala @@ -0,0 +1,7 @@ +package com.twitter.tweet_mixer.feature + +object EntityTypes { + type UserId = Long + type TweetId = Long + type SearchQuery = String +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/FromInNetworkSourceFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/FromInNetworkSourceFeature.scala new file mode 100644 index 000000000..d9b76e71e --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/FromInNetworkSourceFeature.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature + +object FromInNetworkSourceFeature extends Feature[TweetCandidate, Boolean] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/HydraScoreFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/HydraScoreFeature.scala new file mode 100644 index 000000000..7540fa962 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/HydraScoreFeature.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature + +object HydraScoreFeature extends Feature[TweetCandidate, Map[String, Double]] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/InReplyToTweetIdFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/InReplyToTweetIdFeature.scala new file mode 100644 index 000000000..811ebeb0f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/InReplyToTweetIdFeature.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature + +object InReplyToTweetIdFeature extends Feature[TweetCandidate, Option[Long]] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/LanguageCodeFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/LanguageCodeFeature.scala new file mode 100644 index 000000000..bf1edf82b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/LanguageCodeFeature.scala @@ -0,0 +1,8 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure + +object LanguageCodeFeature extends FeatureWithDefaultOnFailure[TweetCandidate, Option[String]] { + override val defaultValue: Option[String] = None +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/LowSignalUserFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/LowSignalUserFeature.scala new file mode 100644 index 000000000..7bf25343b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/LowSignalUserFeature.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object LowSignalUserFeature extends Feature[PipelineQuery, Boolean] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/MediaMetadataFeatures.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/MediaMetadataFeatures.scala new file mode 100644 index 000000000..d371cef22 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/MediaMetadataFeatures.scala @@ -0,0 +1,12 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object MediaClusterIdFeature extends Feature[TweetCandidate, Option[Long]] + +object ImpressedMediaIds extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { + override val defaultValue: Seq[Long] = Seq.empty +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/PredictionRequestIdFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/PredictionRequestIdFeature.scala new file mode 100644 index 000000000..d0388cc08 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/PredictionRequestIdFeature.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature + +object PredictionRequestIdFeature extends Feature[TweetCandidate, Option[Long]] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/RealGraphInNetworkScoresFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/RealGraphInNetworkScoresFeature.scala new file mode 100644 index 000000000..852844526 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/RealGraphInNetworkScoresFeature.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object RealGraphInNetworkScoresFeature extends Feature[PipelineQuery, Map[Long, Double]] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/RequestCountryPlaceIdFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/RequestCountryPlaceIdFeature.scala new file mode 100644 index 000000000..87c30145f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/RequestCountryPlaceIdFeature.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object RequestCountryPlaceIdFeature extends Feature[PipelineQuery, Long] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/ScoreFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/ScoreFeature.scala new file mode 100644 index 000000000..0e1a026ab --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/ScoreFeature.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object ScoreFeature extends Feature[PipelineQuery, Double] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SignalInfo.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SignalInfo.scala new file mode 100644 index 000000000..2f648aa88 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SignalInfo.scala @@ -0,0 +1,11 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.usersignalservice.thriftscala._ +import com.twitter.util.Time + +case class SignalInfo( + signalEntity: SignalEntity, + signalType: SignalType, + sourceEventTime: Option[Time], + authorId: Option[Long] = None, +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SourceSignalFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SourceSignalFeature.scala new file mode 100644 index 000000000..57d9fe7e2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SourceSignalFeature.scala @@ -0,0 +1,8 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object SourceSignalFeature extends Feature[PipelineQuery, Long] + +object SearcherRealtimeHistorySourceSignalFeature extends Feature[PipelineQuery, String] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SourceTweetIdFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SourceTweetIdFeature.scala new file mode 100644 index 000000000..13a511981 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/SourceTweetIdFeature.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature + +object SourceTweetIdFeature extends Feature[TweetCandidate, Option[Long]] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TopicTweetScore.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TopicTweetScore.scala new file mode 100644 index 000000000..a3081a6af --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TopicTweetScore.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature + +object TopicTweetScore extends Feature[TweetCandidate, Option[Double]] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TripTweetScore.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TripTweetScore.scala new file mode 100644 index 000000000..677442dd5 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TripTweetScore.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature + +object TripTweetScore extends Feature[TweetCandidate, Option[Double]] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TweetInfoFeatures.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TweetInfoFeatures.scala new file mode 100644 index 000000000..a604dcf5e --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TweetInfoFeatures.scala @@ -0,0 +1,48 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature + +object TweetBooleanInfoFeature extends Feature[TweetCandidate, Option[Int]] + +object AuthorIdFeature extends Feature[TweetCandidate, Option[Long]] + +object MediaIdFeature extends Feature[TweetCandidate, Option[Long]] + +object TweetInfoFeatures { + val IsReply = "IsReply" + val HasVideo = "HasVideo" + val HasUrl = "HasUrl" + val HasMultipleMedia = "HasMultipleMedia" + val IsHighMediaResolution = "IsHighMediaResolution" + val IsLongVideo = "IsLongVideo" + val IsLandscapeVideo = "IsLandscapeVideo" + val IsRetweet = "IsRetweet" + val HasImage = "HasImage" + val IsShortFormVideo = "IsShortFormVideo" + val IsLongFormVideo = "IsLongFormVideo" + + val TweetInfoIndex: Map[String, Int] = Map( + IsReply -> 0, + HasVideo -> 1, + HasUrl -> 2, + HasMultipleMedia -> 3, + IsHighMediaResolution -> 4, + IsLongVideo -> 5, + IsLandscapeVideo -> 6, + IsRetweet -> 7, + HasImage -> 8, + IsShortFormVideo -> 9, + IsLongFormVideo -> 10 + ) + + def setFeature(featureName: String, bitmap: Int): Int = { + val i = TweetInfoIndex.getOrElse(featureName, 31) //getOrElse should never reach else stage + bitmap | (1 << i) + } + + def isFeatureSet(feature: String, bitmap: Int): Boolean = { + val i = TweetInfoIndex.getOrElse(feature, 31) //getOrElse should never reach else stage + (bitmap & (1 << i)) != 0 + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TweetTopicIdFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TweetTopicIdFeature.scala new file mode 100644 index 000000000..68f559695 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/TweetTopicIdFeature.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature + +object TweetTopicIdFeature extends Feature[TweetCandidate, Long] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/USSFeatures.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/USSFeatures.scala new file mode 100644 index 000000000..44e4a4b5f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/USSFeatures.scala @@ -0,0 +1,217 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.conversions.DurationOps._ +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.tweet_mixer.feature.EntityTypes._ +import com.twitter.usersignalservice.thriftscala.SignalType +import com.twitter.util.Time + +class USSFeature[T] extends FeatureWithDefaultOnFailure[PipelineQuery, Map[T, Seq[SignalInfo]]] { + val defaultValue = Map.empty[T, Seq[SignalInfo]] + def getValue(query: PipelineQuery): Map[T, Seq[SignalInfo]] = { + query.features + .getOrElse(FeatureMap.empty) + .getOrElse(this, defaultValue) + } +} + +object TweetFavorites extends USSFeature[TweetId] +object TweetFeedbackRelevant extends USSFeature[TweetId] +object TweetFeedbackNotrelevant extends USSFeature[TweetId] +object Retweets extends USSFeature[TweetId] +object TweetReplies extends USSFeature[TweetId] +object TweetBookmarks extends USSFeature[TweetId] +object OriginalTweets extends USSFeature[TweetId] +object AccountFollows extends USSFeature[UserId] +object RepeatedProfileVisits extends USSFeature[UserId] +object TweetShares extends USSFeature[TweetId] +object TweetPhotoExpands extends USSFeature[TweetId] +object SearchTweetClicks extends USSFeature[TweetId] +object ProfileTweetClicks extends USSFeature[TweetId] +object TweetVideoOpens extends USSFeature[TweetId] +object TweetDetailGoodClick1Min extends USSFeature[TweetId] +object VideoViewTweets extends USSFeature[TweetId] +object VideoViewVisibilityFilteredTweets extends USSFeature[TweetId] +object VideoViewVisibility75FilteredTweets extends USSFeature[TweetId] +object VideoViewVisibility100FilteredTweets extends USSFeature[TweetId] +object VideoViewHighResolutionFilteredTweets extends USSFeature[TweetId] +object ImmersiveVideoViewTweets extends USSFeature[TweetId] +object MediaImmersiveVideoViewTweets extends USSFeature[TweetId] +object TvVideoViewTweets extends USSFeature[TweetId] +object WatchTimeTweets extends USSFeature[TweetId] +object ImmersiveWatchTimeTweets extends USSFeature[TweetId] +object TvWatchTimeTweets extends USSFeature[TweetId] +object MediaImmersiveWatchTimeTweets extends USSFeature[TweetId] +object RecentNotifications extends USSFeature[TweetId] +object AccountBlocks extends USSFeature[UserId] +object AccountMutes extends USSFeature[UserId] +object TweetReports extends USSFeature[TweetId] +object TweetDontLikes extends USSFeature[TweetId] +object SearcherRealtimeHistory extends USSFeature[SearchQuery] +object NegativeSourceSignal extends USSFeature[TweetId] +object HighQualitySourceTweet extends USSFeature[TweetId] +object HighQualitySourceUser extends USSFeature[UserId] +object HighQualitySourceTweetV2 extends USSFeature[TweetId] +object HighQualitySourceUserV2 extends USSFeature[UserId] + +object USSFeatures { + // Not used by default, testing use for VideoView signal + final val MaxSignalAge = 24.hours + + final val TweetFeatures = Set( + TweetFavorites, + Retweets, + TweetReplies, + TweetBookmarks, + OriginalTweets, + TweetShares, + TweetPhotoExpands, + SearchTweetClicks, + ProfileTweetClicks, + TweetVideoOpens, + VideoViewTweets, + VideoViewVisibilityFilteredTweets, + VideoViewVisibility75FilteredTweets, + VideoViewVisibility100FilteredTweets, + VideoViewHighResolutionFilteredTweets, + ImmersiveVideoViewTweets, + ImmersiveWatchTimeTweets, + RecentNotifications, + TweetDetailGoodClick1Min, + TweetFeedbackRelevant, + HighQualitySourceTweet, + HighQualitySourceTweetV2 + ) + + final val ExplicitEngagementTweetFeatures = Set( + TweetFavorites, + Retweets, + TweetReplies, + TweetBookmarks, + OriginalTweets, + TweetShares, + TweetFeedbackNotrelevant + ) + + final val NonVideoTweetFeatures = Set( + TweetFavorites, + Retweets, + TweetReplies, + TweetBookmarks, + OriginalTweets, + TweetShares, + RecentNotifications, + TweetFeedbackRelevant + ) + + final val VideoTweetFeatures: Set[USSFeature[Long]] = Set(VideoViewTweets) + final val ImmersiveVideoTweetFeatures: Set[USSFeature[Long]] = + Set(ImmersiveVideoViewTweets, ImmersiveWatchTimeTweets) + final val ProducerFeatures = + Set(AccountFollows, RepeatedProfileVisits, HighQualitySourceUser, HighQualitySourceUserV2) + final val HighQualityTweetFeatures: Set[USSFeature[Long]] = Set(HighQualitySourceTweet) + final val NegativeFeatures = Set( + AccountMutes, + AccountBlocks, + TweetReports, + TweetDontLikes, + TweetFeedbackNotrelevant, + NegativeSourceSignal + ) + final val NegativeFeaturesTweetBased = + Set(TweetReports, TweetDontLikes, TweetFeedbackNotrelevant) + final val SearcherRealtimeHistoryFeatures: Set[USSFeature[SearchQuery]] = + Set(SearcherRealtimeHistory) + + private val DefaultPriority = 3 + private def getSignalPriority(signalInfo: SignalInfo): Int = { + signalInfo.signalType match { + case SignalType.HighQualitySourceTweet => 4 + case SignalType.HighQualitySourceUser => 4 + case _ => 3 + } + } + + def getPriority(signalInfos: Seq[SignalInfo]): Int = { + if (signalInfos.isEmpty) DefaultPriority + else signalInfos.map(getSignalPriority).max + } + + private def getWeightedSignalPriority(signalInfo: SignalInfo): (Option[SignalType], Double) = { + signalInfo.signalType match { + case SignalType.TweetFavorite => (Some(SignalType.TweetFavorite), 0.2) + case SignalType.TweetBookmarkV1 => (Some(SignalType.TweetBookmarkV1), 0.15) + case SignalType.TweetShareV1 => (Some(SignalType.TweetShareV1), 0.15) + case SignalType.Retweet => (Some(SignalType.Retweet), 0.15) + case SignalType.Reply => (Some(SignalType.Reply), 0.15) + case _ => (None, 0.2) + } + } + + def getWeightedPriority( + signalInfos: Seq[SignalInfo] + ): (Option[SignalType], Double) = { + signalInfos + .map(getWeightedSignalPriority).maxBy { + case (_, weight) => weight + } + } + + def getSignals[T]( + query: PipelineQuery, + features: Set[USSFeature[T]], + filterNegative: Boolean = true, + filterOldSignals: Boolean = false + ): Seq[T] = { + // Value of each feature is a map of [EntityId, Seq of Timestamps], here _._1 signifies the entityId + val negativeSignals: Map[T, Time] = NegativeFeatures + .flatMap(_.getValue(query)) + .map { + case (signalId, signalInfos) => + ( + signalId.asInstanceOf[T], + signalInfos.flatMap(_.sourceEventTime).foldLeft(Time.Zero)(_ max _) + ) + }.toMap + + val positiveSignals = features.flatMap { + _.getValue(query) + .map { + case (signalId, signalInfos) => + val mostRecentTimeStamp = + signalInfos.flatMap(_.sourceEventTime).foldLeft(Time.Zero)(_ max _) + (signalId, Option(signalInfos.headOption.flatMap(_.authorId)), mostRecentTimeStamp) + } + } + + val timeFilteredSignals = if (filterOldSignals) { + positiveSignals.filter { case (_, _, time) => Time.now - time < MaxSignalAge } + } else positiveSignals + + val filteredSignals = if (filterNegative) { + timeFilteredSignals + .filterNot { + case (signalId, authorIdOpt, _) => + negativeSignals.contains(signalId) || + authorIdOpt.exists(authorId => negativeSignals.contains(authorId.asInstanceOf[T])) + } + // Filter signalId that has a negative engagement that happens later than this signalId + } else timeFilteredSignals + + filteredSignals.toSeq + .sortWith(_._3 > _._3) // Sort by MostRecentTimestamp + .map(_._1) // Get only keys which are entityIds (tweet/userIds) + } + + def getSignalsWithInfo[T]( + query: PipelineQuery, + features: Set[USSFeature[T]] + ): Map[T, Seq[SignalInfo]] = { + features + .flatMap(_.getValue(query)) + .groupBy(_._1) + .mapValues(_.map(_._2).toSeq.flatten) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/UserTopicIdsFeature.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/UserTopicIdsFeature.scala new file mode 100644 index 000000000..4c3d1679d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature/UserTopicIdsFeature.scala @@ -0,0 +1,6 @@ +package com.twitter.tweet_mixer.feature + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +object UserTopicIdsFeature extends Feature[PipelineQuery, Seq[Long]] diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/BUILD.bazel new file mode 100644 index 000000000..34f0e9d10 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/BUILD.bazel @@ -0,0 +1,29 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", + "strato/config/columns/content_understanding:content_understanding-strato-client", + "strato/config/columns/unified-counter/service:service-strato-client", + "strato/config/columns/videoRecommendations/twitterClip:twitterClip-strato-client", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + "viewcounts/src/main/thrift/com/twitter/viewcounts:viewcounts-scala", + ], + exports = [ + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/TweetMixerFunctionalComponents.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/TweetMixerFunctionalComponents.scala new file mode 100644 index 000000000..e4e536d24 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/TweetMixerFunctionalComponents.scala @@ -0,0 +1,333 @@ +package com.twitter.tweet_mixer.functional_component + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.selector._ +import com.twitter.product_mixer.core.functional_component.common.AllExceptPipelines +import com.twitter.product_mixer.core.functional_component.common.AllPipelines +import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.request.HasExcludedIds +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.spam.rtf.thriftscala.SafetyLevel +import com.twitter.stitch.cache.AsyncValueCache +import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} +import com.twitter.strato.client.Client +import com.twitter.strato.generated.client.content_understanding.ColdStartPostsMetadataMhClientColumn +import com.twitter.strato.generated.client.unified_counter.service.UecAggTotalOnTweetClientColumn +import com.twitter.strato.generated.client.videoRecommendations.twitterClip.TwitterClipClusterIdMhClientColumn +import com.twitter.timelines.configapi.FSEnumParam +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.feature.SourceSignalFeature +import com.twitter.tweet_mixer.feature.USSFeatures +import com.twitter.tweet_mixer.functional_component.filter.ImpressedTweetsFilter +import com.twitter.tweet_mixer.functional_component.filter.TweetVisibilityAndReplyFilter +import com.twitter.tweet_mixer.functional_component.hydrator._ +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.BlendingEnum +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.BlendingParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableMediaMetadataCandidateFeatureHydrator +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.GrokFilterFeatureHydratorEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ViewCountInfoOnTweetFeatureHydratorEnabled +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.usersignalservice.thriftscala.SignalType +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import scala.util.Random + +@Singleton +class TweetMixerFunctionalComponents @Inject() ( + ussQueryFeatureHydrator: USSQueryFeatureHydrator, + highQualitySourceSignalQueryFeatureHydrator: HighQualitySourceSignalQueryFeatureHydrator, + requestPlaceIdsQueryFeatureHydrator: RequestPlaceIdsQueryFeatureHydrator, + tweetypieStitchClient: TweetypieStitchClient, + twitterClipClusterIdMhClientColumn: TwitterClipClusterIdMhClientColumn, + coldStartPostsMetadataMhClientColumn: ColdStartPostsMetadataMhClientColumn, + uecAggTotalOnTweetClientColumn: UecAggTotalOnTweetClientColumn, + userTopicIdsFeatureHydrator: UserTopicIdsFeatureHydrator, + memcache: MemcacheStitchClient, + @Named(ModuleNames.TweeypieInMemCache) + inMemoryCache: AsyncValueCache[java.lang.Long, Option[(Int, Long, Long)]], + @Named(ModuleNames.MediaMetadataInMemCache) + mediaMetadataInMemCache: AsyncValueCache[java.lang.Long, Option[Long]], + @Named(ModuleNames.GrokFilterInMemCache) + grokFilterInMemCache: AsyncValueCache[java.lang.Long, Boolean], + statsReceiver: StatsReceiver, + @Named("StratoClientWithModerateTimeout") stratoClient: Client) { + + private val identifierPrefix: String = "HomeRecommendedTweets" + + private val tweetypieCandidateFeatureHydrator = new TweetypieCandidateFeatureHydrator( + tweetypieStitchClient = tweetypieStitchClient, + safetyLevelPredicate = _ => SafetyLevel.Recommendations, + memcache = memcache, + inMemoryCache = inMemoryCache, + statsReceiver = statsReceiver, + stratoClient + ) + + private val mediaMetadataCandidateFeatureHydrator = new MediaMetadataCandidateFeatureHydrator( + twitterClipClusterIdMhClientColumn = twitterClipClusterIdMhClientColumn, + memcache = memcache, + inMemoryCache = mediaMetadataInMemCache, + statsReceiver = statsReceiver + ) + + private val uecAggTweetTotalFeatureHydrator = new UecAggTweetTotalFeatureHydrator( + uecAggTotalOnTweetClientColumn = uecAggTotalOnTweetClientColumn, + candidatePipelinesToInclude = Some( + Set( + CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationEmbeddingSimilarity), + CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationEmbeddingSimilarityTierTwo), + CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRTweetTweet), + CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRTweetTweetTierTwo), + CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRUserTweet), + CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRUserTweetTierTwo) + ) + ), + statsReceiver = statsReceiver + ) + + private val grokFilterFeatureHydrator = new GrokFilterFeatureHydrator( + coldStartPostsMetadataMhClientColumn = coldStartPostsMetadataMhClientColumn, + statsReceiver = statsReceiver, + memcache = memcache, + inMemoryCache = grokFilterInMemCache + ) + + val globalQueryFeatureHydrators: Seq[QueryFeatureHydrator[PipelineQuery]] = Seq( + ussQueryFeatureHydrator, + highQualitySourceSignalQueryFeatureHydrator, + requestPlaceIdsQueryFeatureHydrator, + userTopicIdsFeatureHydrator + ) + + val globalCandidateFeatureHydrators: Seq[ + BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + ] = Seq( + tweetypieCandidateFeatureHydrator, + SignalInfoCandidateFeatureHydrator, + ParamGatedBulkCandidateFeatureHydrator( + EnableMediaMetadataCandidateFeatureHydrator, + mediaMetadataCandidateFeatureHydrator + ), + ParamGatedBulkCandidateFeatureHydrator( + ViewCountInfoOnTweetFeatureHydratorEnabled, + uecAggTweetTotalFeatureHydrator + ), + ParamGatedBulkCandidateFeatureHydrator( + GrokFilterFeatureHydratorEnabled, + grokFilterFeatureHydrator + ) + ) + + def globalFilters( + candidatePipelinesToExclude: Set[CandidatePipelineIdentifier] = Set.empty + ): Seq[Filter[PipelineQuery with HasExcludedIds, TweetCandidate]] = Seq( + TweetVisibilityAndReplyFilter(candidatePipelinesToExclude), + ImpressedTweetsFilter + ) + + private val roundRobinBucketer: Bucketer[(ComponentIdentifier, Long)] = + (candidateWithDetails: CandidateWithDetails) => + ( + candidateWithDetails.source, + candidateWithDetails.features.getOrElse(SourceSignalFeature, -1L), + ) + + private val roundRobinSourceSignalBucketer: Bucketer[Long] = + (candidateWithDetails: CandidateWithDetails) => + candidateWithDetails.features.getOrElse(SourceSignalFeature, -1L) + + private val roundRobinSourceBucketer: Bucketer[ComponentIdentifier] = + (candidateWithDetails: CandidateWithDetails) => candidateWithDetails.source + + private val signalPriorityBucketer: Bucketer[(ComponentIdentifier, Long, Int)] = + (candidateWithDetails: CandidateWithDetails) => + ( + candidateWithDetails.source, + candidateWithDetails.features.getOrElse(SourceSignalFeature, -1L), + USSFeatures.getPriority( + candidateWithDetails.features.getOrElse(SignalInfoFeature, Seq.empty) + ) + ) + + private val getPriorityFn: ((ComponentIdentifier, Long, Int)) => Int = bucket => bucket._3 + + private val weightedPriorityBucketer: Bucketer[ + (ComponentIdentifier, Long, (Option[SignalType], Double)) + ] = + (candidateWithDetails: CandidateWithDetails) => + ( + candidateWithDetails.source, + candidateWithDetails.features.getOrElse(SourceSignalFeature, -1L), + USSFeatures.getWeightedPriority( + candidateWithDetails.features.getOrElse(SignalInfoFeature, Seq.empty) + ) + ) + + private val getWeightedPriorityFn: ( + (ComponentIdentifier, Long, (Option[SignalType], Double)) + ) => (Option[SignalType], Double) = + bucket => bucket._3 + + private def paramEnumGated[T <: Enumeration]( + selectors: Seq[Selector[PipelineQuery]], + enumParam: FSEnumParam[T], + matchingValue: T#Value + ): Seq[Selector[PipelineQuery]] = + SelectConditionally(selectors, (query, _, _) => query.params(enumParam) == matchingValue) + + private def paramEnumGated[T <: Enumeration]( + selector: Selector[PipelineQuery], + enumParam: FSEnumParam[T], + matchingValue: T#Value + ): Selector[PipelineQuery] = + SelectConditionally(selector, (query, _, _) => query.params(enumParam) == matchingValue) + + private def paramNotEnumGated[T <: Enumeration]( + selector: Selector[PipelineQuery], + enumParam: FSEnumParam[T], + matchingValue: T#Value + ): Selector[PipelineQuery] = + SelectConditionally(selector, (query, _, _) => query.params(enumParam) != matchingValue) + + val roundRobinSelector = paramEnumGated[BlendingEnum.type]( + InsertAppendWeaveResults(AllPipelines, roundRobinBucketer), + BlendingParam, + BlendingEnum.RoundRobinBlending + ) + + def roundRobinConditionSelectors( + selectors: Seq[Selector[PipelineQuery]] + ): Seq[Selector[PipelineQuery]] = { + paramEnumGated[BlendingEnum.type]( + selectors, + BlendingParam, + BlendingEnum.RoundRobinBlending + ) + } + + def exclusiveRoundRobinSelector(pipelinesToExclude: Set[CandidatePipelineIdentifier]) = { + paramEnumGated[BlendingEnum.type]( + InsertAppendWeaveResults( + AllExceptPipelines(pipelinesToExclude = pipelinesToExclude), + roundRobinBucketer + ), + BlendingParam, + BlendingEnum.RoundRobinBlending + ) + } + + def inclusiveRoundRobinSelector(pipelinesToInclude: Set[CandidatePipelineIdentifier]) = { + paramEnumGated[BlendingEnum.type]( + InsertAppendWeaveResults( + SpecificPipelines(pipelines = pipelinesToInclude), + roundRobinBucketer + ), + BlendingParam, + BlendingEnum.RoundRobinBlending + ) + } + + def exclusiveRoundRobinSelectorWithParam( + pipelinesToExcludeMap: Map[CandidatePipelineIdentifier, FSParam[Boolean]] + ): SelectConditionallyWithFn[PipelineQuery] = { + val selectorFn: PipelineQuery => Selector[PipelineQuery] = query => { + val pipelinesToExclude = pipelinesToExcludeMap.filter { + case (_, enableParam) => query.params(enableParam) + }.keySet + InsertAppendWeaveResults( + AllExceptPipelines(pipelinesToExclude = pipelinesToExclude), + roundRobinBucketer) + } + SelectConditionallyWithFn( + selectorFn, + ( + query: PipelineQuery, + _, + _ + ) => query.params(BlendingParam) == BlendingEnum.RoundRobinBlending, + AllPipelines + ) + } + + def signalPrioritySelector(pipelinesToExclude: Set[CandidatePipelineIdentifier] = Set.empty) = { + paramEnumGated[BlendingEnum.type]( + InsertAppendPriorityWeaveResults( + AllExceptPipelines(pipelinesToExclude = pipelinesToExclude), + signalPriorityBucketer, + getPriorityFn), + BlendingParam, + BlendingEnum.SignalPriorityBlending + ) + } + + def weightedSignalPrioritySelector( + pipelinesToExclude: Set[CandidatePipelineIdentifier] = Set.empty + ): Selector[PipelineQuery] = { + paramEnumGated[BlendingEnum.type]( + InsertAppendWeightedSignalPriorityWeaveResults( + AllExceptPipelines(pipelinesToExclude = pipelinesToExclude), + weightedPriorityBucketer, + getWeightedPriorityFn, + random = new Random(0) + ), + BlendingParam, + BlendingEnum.WeightedPriorityBlending + ) + } + + val roundRobinSourceSignalSelector = paramEnumGated[BlendingEnum.type]( + InsertAppendWeaveResults(AllPipelines, roundRobinSourceSignalBucketer), + BlendingParam, + BlendingEnum.RoundRobinSourceSignalBlending + ) + + val roundRobinSourceSelector = paramEnumGated[BlendingEnum.type]( + InsertAppendWeaveResults(AllPipelines, roundRobinSourceBucketer), + BlendingParam, + BlendingEnum.RoundRobinSourceBlending + ) + + val rankingOnlySelector = paramEnumGated[BlendingEnum.type]( + InsertAppendResults(AllPipelines), + BlendingParam, + BlendingEnum.RankingOnly + ) + + def exclusiveRankingOnlySelector(pipelinesToExclude: Set[CandidatePipelineIdentifier]) = { + paramEnumGated[BlendingEnum.type]( + InsertAppendResults(AllExceptPipelines(pipelinesToExclude = pipelinesToExclude)), + BlendingParam, + BlendingEnum.RankingOnly + ) + } + + val rankingOnlyDeduplicationSelector = paramEnumGated[BlendingEnum.type]( + DropDuplicateResults(duplicationKey = IdDuplicationKey), + BlendingParam, + BlendingEnum.RankingOnly + ) + + val rankingOnlyNotDeduplicationSelector = paramNotEnumGated[BlendingEnum.type]( + DropDuplicateResults(duplicationKey = IdDuplicationKey), + BlendingParam, + BlendingEnum.RankingOnly + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/BUILD.bazel new file mode 100644 index 000000000..9a4793b8f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/BUILD.bazel @@ -0,0 +1,18 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", + "strato/config/src/thrift/com/twitter/strato/graphql/unified_counter:graphql-unified-counter-scala", + "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/imv_related_tweets/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/GrokFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/GrokFilter.scala new file mode 100644 index 000000000..609e25c8c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/GrokFilter.scala @@ -0,0 +1,33 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.functional_component.hydrator.GrokFilterFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableGrokFilter + +object GrokFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("GrokFilter") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableGrokFilter) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (kept, removed) = candidates.partition { candidate => + !candidate.features.getOrElse(GrokFilterFeature, false) + } + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ImpressedTweetsBloomFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ImpressedTweetsBloomFilter.scala new file mode 100644 index 000000000..7ea8bb415 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ImpressedTweetsBloomFilter.scala @@ -0,0 +1,58 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter +import com.twitter.tweet_mixer.feature.SourceTweetIdFeature +import com.twitter.tweet_mixer.feature.USSFeatures +import com.twitter.tweet_mixer.functional_component.hydrator.ImpressionBloomFilterFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableImpressionBloomFilterHydrator + +/** + * Filter out users' previously seen tweets from Bloom Filter store + */ +object ImpressedTweetsBloomFilter + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("ImpressedTweetsBloom") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = { + query.params(EnableImpressionBloomFilterHydrator) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val bloomFilterSeq = query.features.map(_.get(ImpressionBloomFilterFeature)).get + val tweetSignals = USSFeatures.getSignals[Long](query, USSFeatures.TweetFeatures) + val tweetIdAndSourceIds = candidates.flatMap { candidate => + val tweetId = candidate.candidate.id + val sourceTweetId = + candidate.features.getOrElse(SourceTweetIdFeature, None).getOrElse(tweetId) + Seq(tweetId, sourceTweetId) + }.distinct + val seenTweetIds = + (ImpressionBloomFilter.extractSeenTweetIds( + tweetIdAndSourceIds, + bloomFilterSeq) ++ tweetSignals).toSet + + val (kept, removed) = candidates.partition { candidate => + val tweetId = candidate.candidate.id + val sourceTweetId = + candidate.features.getOrElse(SourceTweetIdFeature, None).getOrElse(tweetId) + !seenTweetIds.contains(tweetId) && !seenTweetIds.contains(sourceTweetId) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ImpressedTweetsFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ImpressedTweetsFilter.scala new file mode 100644 index 000000000..e3d3a1474 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ImpressedTweetsFilter.scala @@ -0,0 +1,38 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.HasExcludedIds +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature._ + +/** + * Applies impressed tweets filter using USS signals and excluded TweetIds + */ +object ImpressedTweetsFilter extends Filter[PipelineQuery with HasExcludedIds, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("ImpressedTweets") + + override def apply( + query: PipelineQuery with HasExcludedIds, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val tweetSignals = USSFeatures.getSignals[Long](query, USSFeatures.TweetFeatures) + val impressedTweets = query.excludedIds ++ tweetSignals.toSet + val (kept, removed) = + candidates.partition { candidate => + val tweetId = candidate.candidate.id + val sourceTweetId = + candidate.features.getOrElse(SourceTweetIdFeature, None).getOrElse(tweetId) + !impressedTweets.contains(tweetId) && !impressedTweets.contains(sourceTweetId) + } + + val filterResult = + FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)) + Stitch.value(filterResult) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsLongFormVideoFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsLongFormVideoFilter.scala new file mode 100644 index 000000000..dfd1fd25f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsLongFormVideoFilter.scala @@ -0,0 +1,54 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.TweetBooleanInfoFeature +import com.twitter.tweet_mixer.feature.TweetInfoFeatures._ +import com.twitter.tweet_mixer.model.request.HasVideoType +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableLongFormVideoFilter +import com.twitter.tweet_mixer.{thriftscala => t} + +/** + * Filters out tweets that are not long videos for the given set of candidate pipelines + */ +case class IsLongFormVideoFilter( + candidatePipelinesToExclude: Set[CandidatePipelineIdentifier] = Set.empty) + extends Filter[PipelineQuery with HasVideoType, TweetCandidate] + with Filter.Conditionally[PipelineQuery with HasVideoType, TweetCandidate] + with ShouldIgnoreCandidatePipelinesFilter { + override val identifier: FilterIdentifier = FilterIdentifier("IsLongFormVideo") + + override def onlyIf( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = { + query.params(EnableLongFormVideoFilter) || query.videoType.contains(t.VideoType.LongForm) + } + + def isLongFormVideo(features: FeatureMap): Boolean = { + val tweetBooleanInfo = features.get(TweetBooleanInfoFeature).getOrElse(0) + isFeatureSet(IsLongFormVideo, tweetBooleanInfo) + } + + override def apply( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (kept, removed) = candidates + .partition { candidate => isLongFormVideo(candidate.features) || shouldIgnore(candidate) } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsPortraitVideoFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsPortraitVideoFilter.scala new file mode 100644 index 000000000..3beb90c4f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsPortraitVideoFilter.scala @@ -0,0 +1,53 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.TweetBooleanInfoFeature +import com.twitter.tweet_mixer.feature.TweetInfoFeatures._ +import com.twitter.tweet_mixer.model.request.HasVideoType +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnablePortraitVideoFilter +import com.twitter.tweet_mixer.{thriftscala => t} + +/** + * Filters out tweets that are not portrait videos for the given set of candidate pipelines + */ +object IsPortraitVideoFilter + extends Filter[PipelineQuery with HasVideoType, TweetCandidate] + with Filter.Conditionally[PipelineQuery with HasVideoType, TweetCandidate] { + override val identifier: FilterIdentifier = FilterIdentifier("IsPortraitVideo") + + override def onlyIf( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = { + query.params(EnablePortraitVideoFilter) || query.videoType.contains(t.VideoType.ShortForm) + } + + def isPortraitVideo(features: FeatureMap): Boolean = { + val tweetBooleanInfo = features.get(TweetBooleanInfoFeature).getOrElse(0) + val hasVideo = isFeatureSet(HasVideo, tweetBooleanInfo) + val isPortraitVideo = !isFeatureSet(IsLandscapeVideo, tweetBooleanInfo) + hasVideo && isPortraitVideo + } + + override def apply( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (kept, removed) = candidates + .partition { candidate => isPortraitVideo(candidate.features) } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsShortFormVideoFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsShortFormVideoFilter.scala new file mode 100644 index 000000000..0bf5a0322 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsShortFormVideoFilter.scala @@ -0,0 +1,55 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.TweetBooleanInfoFeature +import com.twitter.tweet_mixer.feature.TweetInfoFeatures._ +import com.twitter.tweet_mixer.model.request.HasVideoType +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableShortFormVideoFilter +import com.twitter.tweet_mixer.{thriftscala => t} + +/** + * Filters out tweets that are not short videos for the given set of candidate pipelines + */ +case class IsShortFormVideoFilter( + candidatePipelinesToExclude: Set[CandidatePipelineIdentifier] = Set.empty) + extends Filter[PipelineQuery with HasVideoType, TweetCandidate] + with Filter.Conditionally[PipelineQuery with HasVideoType, TweetCandidate] + with ShouldIgnoreCandidatePipelinesFilter { + override val identifier: FilterIdentifier = FilterIdentifier("IsShortFormVideo") + + override def onlyIf( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = { + query.params(EnableShortFormVideoFilter) || query.videoType.contains(t.VideoType.ShortForm) + } + + def isShortFormVideo(features: FeatureMap): Boolean = { + val tweetBooleanInfo = features.get(TweetBooleanInfoFeature).getOrElse(0) + isFeatureSet(IsShortFormVideo, tweetBooleanInfo) + } + + override def apply( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (kept, removed) = candidates + .partition { candidate => isShortFormVideo(candidate.features) || shouldIgnore(candidate) } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsVideoTweetFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsVideoTweetFilter.scala new file mode 100644 index 000000000..e0436ff6d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/IsVideoTweetFilter.scala @@ -0,0 +1,61 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.TweetBooleanInfoFeature +import com.twitter.tweet_mixer.feature.TweetInfoFeatures._ +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableVideoTweetFilter + +/** + * Filters out tweets that are not videos + */ +case class IsVideoTweetFilter( + candidatePipelinesToInclude: Option[Set[CandidatePipelineIdentifier]] = None, + candidatePipelinesToExclude: Set[CandidatePipelineIdentifier] = Set.empty) + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] + with ShouldIgnoreCandidatePipelinesFilter { + override val identifier: FilterIdentifier = FilterIdentifier("IsVideoTweet") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableVideoTweetFilter) + + def isVideo(features: FeatureMap): Boolean = { + val tweetBooleanInfo = features.get(TweetBooleanInfoFeature).getOrElse(0) + val isReply = isFeatureSet(IsReply, tweetBooleanInfo) + val hasVideo = isFeatureSet(HasVideo, tweetBooleanInfo) + val hasMultipleMedia = isFeatureSet(HasMultipleMedia, tweetBooleanInfo) + val hasUrl = isFeatureSet(HasUrl, tweetBooleanInfo) + val isHighMediaResolution = isFeatureSet(IsHighMediaResolution, tweetBooleanInfo) + val isInCandidateScope = candidatePipelinesToInclude + .map(candidatePipelines => + features.get(CandidatePipelines).exists(candidatePipelines.contains(_))).getOrElse(true) + + (!isReply && !hasMultipleMedia && !hasUrl && hasVideo && isHighMediaResolution) || !isInCandidateScope + } + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (kept, removed) = candidates + .partition { candidate => isVideo(candidate.features) || shouldIgnore(candidate) } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MaxViewCountFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MaxViewCountFilter.scala new file mode 100644 index 000000000..2281882f0 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MaxViewCountFilter.scala @@ -0,0 +1,119 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationMaxViewCountThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationTier2MaxViewCountThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ViewCountInfoOnTweetFeatureHydratorEnabled +import com.twitter.tweet_mixer.functional_component.hydrator.UecAggTweetTotalFeature +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants +import com.twitter.strato.graphql.unified_counter.thriftscala.UecAnalyticsEngagementTypes + +/** + * Filters out tweets that pass max view count for specific candidate pipeline + */ +case class MaxViewCountFilter( + candidatePipelinesToInclude: Option[Set[CandidatePipelineIdentifier]] = None, + candidatePipelinesToExclude: Set[CandidatePipelineIdentifier] = Set.empty) + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] + with ShouldIgnoreCandidatePipelinesFilter { + override val identifier: FilterIdentifier = FilterIdentifier("MaxViewCount") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = { + query.params(ViewCountInfoOnTweetFeatureHydratorEnabled) && + (query.params(ContentExplorationMaxViewCountThreshold) > 0 || + query.params(ContentExplorationTier2MaxViewCountThreshold) > 0) + } + + // Keep it same as home recommended tweets domain marshaller name + private val identifierPrefix: String = "HomeRecommendedTweets" + private val pipelineDrI2i = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRTweetTweet) + private val pipelineDrI2iTierTwo = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRTweetTweetTierTwo) + private val pipelineDrU2i = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRUserTweet) + private val pipelineDrU2iTierTwo = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationDRUserTweetTierTwo) + private val pipelineEmb = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationEmbeddingSimilarity) + private val pipelineEmbTierTwo = CandidatePipelineIdentifier( + identifierPrefix + CandidatePipelineConstants.ContentExplorationEmbeddingSimilarityTierTwo) + + def isInCandidateScope(features: FeatureMap): Boolean = { + candidatePipelinesToInclude + .map(candidatePipelines => + features.get(CandidatePipelines).exists(candidatePipelines.contains(_))).getOrElse(false) + } + + def getSuggestionType(features: FeatureMap): String = { + features.get(CandidatePipelines) match { + case p if p.contains(pipelineDrI2i) => "ForYouContentExplorationDeepRetrievalI2i" + case p if p.contains(pipelineDrI2iTierTwo) => "ForYouContentExplorationTier2DeepRetrievalI2i" + case p if p.contains(pipelineDrU2i) => "ForYouContentExplorationDeepRetrievalI2i" + case p if p.contains(pipelineDrU2iTierTwo) => "ForYouContentExplorationTier2DeepRetrievalI2i" + case p if p.contains(pipelineEmb) => "ForYouContentExploration" + case p if p.contains(pipelineEmbTierTwo) => "ForYouContentExplorationTier2" + case _ => "ForYouContentExploration" + } + } + + def getMaxViewCount(features: FeatureMap, query: PipelineQuery): Int = { + val tier2Pipelines = Set( + pipelineDrI2iTierTwo, + pipelineEmbTierTwo, + pipelineDrU2iTierTwo + ) + + if (features.get(CandidatePipelines).exists(tier2Pipelines.contains(_))) { + query.params(ContentExplorationTier2MaxViewCountThreshold) + } else { + query.params(ContentExplorationMaxViewCountThreshold) + } + } + + def passMaxViewCount(features: FeatureMap, maxViewCount: Int, suggestionType: String): Boolean = { + features + .get(UecAggTweetTotalFeature) + .flatMap { aggTotalResponse => + aggTotalResponse.engagements + .find { metric => + metric.engagementType == UecAnalyticsEngagementTypes.Displayed && + metric.suggestionType.contains(suggestionType) + }.map(_.count) + } + .forall(_ < maxViewCount) // return true if value is missing + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val (kept, removed) = candidates.partition { candidate => + if (isInCandidateScope(candidate.features)) { + val suggestionType = getSuggestionType(candidate.features) + val maxViewCount = getMaxViewCount(candidate.features, query) + passMaxViewCount(candidate.features, maxViewCount, suggestionType) + } else true // keep candidates not in the included pipelines + } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaClusterIdDedupFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaClusterIdDedupFilter.scala new file mode 100644 index 000000000..4682e0cd3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaClusterIdDedupFilter.scala @@ -0,0 +1,56 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.MediaClusterIdFeature +import com.twitter.tweet_mixer.model.request.HasVideoType +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableMediaClusterIdDedupFilter + +case class MediaClusterIdDedupFilter( + candidatePipelinesToExclude: Set[CandidatePipelineIdentifier] = Set.empty) + extends Filter[PipelineQuery with HasVideoType, TweetCandidate] + with Filter.Conditionally[PipelineQuery with HasVideoType, TweetCandidate] + with ShouldIgnoreCandidatePipelinesFilter { + + override val identifier: FilterIdentifier = FilterIdentifier("MediaClusterIdDedup") + + override def onlyIf( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableMediaClusterIdDedupFilter) + + override def apply( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (removedCandidateIds, _) = candidates.foldLeft((Set[Long](), Set[Long]())) { + case ((removeIds, clusterIdsToExclude), candidate) => + if (shouldIgnore(candidate)) { + (removeIds, clusterIdsToExclude + candidate.candidate.id) + } else { + candidate.features.getOrElse(MediaClusterIdFeature, None) match { + case Some(clusterId) => + if (clusterIdsToExclude.contains(clusterId)) { + (removeIds + candidate.candidate.id, clusterIdsToExclude) + } else { + (removeIds, clusterIdsToExclude + clusterId) + } + case _ => (removeIds, clusterIdsToExclude) + } + } + } + + val (removed, kept) = candidates + .map(_.candidate) + .partition(candidate => removedCandidateIds.contains(candidate.id)) + + Stitch.value(FilterResult(kept = kept, removed = removed)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaIdDedupFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaIdDedupFilter.scala new file mode 100644 index 000000000..a411f2d29 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaIdDedupFilter.scala @@ -0,0 +1,56 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.MediaIdFeature +import com.twitter.tweet_mixer.model.request.HasVideoType +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableMediaIdDedupFilter + +case class MediaIdDedupFilter( + candidatePipelinesToExclude: Set[CandidatePipelineIdentifier] = Set.empty) + extends Filter[PipelineQuery with HasVideoType, TweetCandidate] + with Filter.Conditionally[PipelineQuery with HasVideoType, TweetCandidate] + with ShouldIgnoreCandidatePipelinesFilter { + + override val identifier: FilterIdentifier = FilterIdentifier("MediaIdDedup") + + override def onlyIf( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableMediaIdDedupFilter) + + override def apply( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (removedCandidateIds, _) = candidates.foldLeft((Set[Long](), Set[Long]())) { + case ((removeIds, mediaIdsToExclude), candidate) => + if (shouldIgnore(candidate)) { + (removeIds, mediaIdsToExclude + candidate.candidate.id) + } else { + candidate.features.getOrElse(MediaIdFeature, None) match { + case Some(mediaId) => + // Unhydrated media has ID 0 + if (mediaId == 0L) (removeIds, mediaIdsToExclude) + else if (mediaIdsToExclude.contains(mediaId)) + (removeIds + candidate.candidate.id, mediaIdsToExclude) + else (removeIds, mediaIdsToExclude + mediaId) + case _ => (removeIds, mediaIdsToExclude) + } + } + } + + val (removed, kept) = candidates + .map(_.candidate) + .partition(candidate => removedCandidateIds.contains(candidate.id)) + + Stitch.value(FilterResult(kept = kept, removed = removed)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaWatchHistoryFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaWatchHistoryFilter.scala new file mode 100644 index 000000000..ced1692bc --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MediaWatchHistoryFilter.scala @@ -0,0 +1,48 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilterItem +import com.twitter.tweet_mixer.feature.MediaIdFeature +import com.twitter.tweet_mixer.functional_component.hydrator.VideoImpressionBloomFilterFeature +import com.twitter.tweet_mixer.model.request.HasVideoType +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableVideoBloomFilterHydrator + +object MediaWatchHistoryFilter + extends Filter[PipelineQuery with HasVideoType, TweetCandidate] + with Filter.Conditionally[PipelineQuery with HasVideoType, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("MediaWatchHistory") + + override def onlyIf( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = query.params(EnableVideoBloomFilterHydrator) + + override def apply( + query: PipelineQuery with HasVideoType, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val bloomFilters = query.features.map(_.get(VideoImpressionBloomFilterFeature)) match { + case Some(seq) => seq.entries.map(ImpressionBloomFilterItem.fromThrift(_).bloomFilter) + case None => Seq.empty + } + + val (removed, kept) = candidates.partition { candidate => + getMediaId(candidate).exists { mediaId => + bloomFilters.exists(filter => filter.mayContain(mediaId)) + } + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } + + private def getMediaId(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] = { + candidate.features.getOrElse(MediaIdFeature, None) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MinScoreFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MinScoreFilter.scala new file mode 100644 index 000000000..b106434bc --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/MinScoreFilter.scala @@ -0,0 +1,80 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.ScoreFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetANNScoreThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetANNScoreMaxThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetEmbeddingANNScoreThreshold +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants + +/** + * Filters out tweets that pass min score for specific candidate pipeline + */ +case class MinScoreFilter( + candidatePipelinesToInclude: Option[Set[CandidatePipelineIdentifier]] = None, + candidatePipelinesToExclude: Set[CandidatePipelineIdentifier] = Set.empty) + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] + with ShouldIgnoreCandidatePipelinesFilter { + + override val identifier: FilterIdentifier = FilterIdentifier("MinScore") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = { + query.params(DeepRetrievalTweetTweetANNScoreThreshold) > 0.0 || + query.params(DeepRetrievalTweetTweetANNScoreMaxThreshold) < 1.0 || + query.params(DeepRetrievalTweetTweetEmbeddingANNScoreThreshold) > 0.0 + } + + def isInCandidateScope(features: FeatureMap): Boolean = { + candidatePipelinesToInclude + .map(candidatePipelines => + features.get(CandidatePipelines).exists(candidatePipelines.contains(_))).getOrElse(false) + } + + def passMinScore(features: FeatureMap, minScore: Double): Boolean = + features.get(ScoreFeature) >= minScore + + def passMaxScore(features: FeatureMap, maxScore: Double): Boolean = + features.get(ScoreFeature) <= maxScore + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val minScoreThreshold = query.params(DeepRetrievalTweetTweetANNScoreThreshold) + val maxScoreThreshold = query.params(DeepRetrievalTweetTweetANNScoreMaxThreshold) + val embMinScoreThreshold = query.params(DeepRetrievalTweetTweetEmbeddingANNScoreThreshold) + val embPipelineId = CandidatePipelineIdentifier( + "HomeRecommendedTweets" + CandidatePipelineConstants.DeepRetrievalTweetTweetEmbeddingSimilarity) + + val (kept, removed) = candidates.partition { candidate => + if (isInCandidateScope(candidate.features)) { + val minScore = + if (candidate.features.get(CandidatePipelines).contains(embPipelineId)) + embMinScoreThreshold + else minScoreThreshold + passMinScore(candidate.features, minScore) && + passMaxScore(candidate.features, maxScoreThreshold) + } else true // keep candidates not in the included pipelines + } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ShouldIgnoreCandidatePipelinesFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ShouldIgnoreCandidatePipelinesFilter.scala new file mode 100644 index 000000000..57a3a53ec --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/ShouldIgnoreCandidatePipelinesFilter.scala @@ -0,0 +1,16 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines + +trait ShouldIgnoreCandidatePipelinesFilter { + def candidatePipelinesToExclude: Set[CandidatePipelineIdentifier] + + def shouldIgnore(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = { + candidate.features + .get(CandidatePipelines) + .exists(pipeline => candidatePipelinesToExclude.contains(pipeline)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/TweetVisibilityAndReplyFilter.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/TweetVisibilityAndReplyFilter.scala new file mode 100644 index 000000000..cab68d4db --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter/TweetVisibilityAndReplyFilter.scala @@ -0,0 +1,62 @@ +package com.twitter.tweet_mixer.functional_component.filter + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.FromInNetworkSourceFeature +import com.twitter.tweet_mixer.feature.TweetBooleanInfoFeature +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.IsReply +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.isFeatureSet +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableDebugMode + +/** + * Filters out tweets that has reply column unfilled + */ +case class TweetVisibilityAndReplyFilter( + candidatePipelinesToExclude: Set[CandidatePipelineIdentifier] = Set.empty) + extends Filter[PipelineQuery, TweetCandidate] + with Filter.Conditionally[PipelineQuery, TweetCandidate] + with ShouldIgnoreCandidatePipelinesFilter { + + override val identifier: FilterIdentifier = FilterIdentifier("TweetVisibilityPolicy") + + override def onlyIf( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Boolean = !query.params(EnableDebugMode) + + private def isVisible(tweetBooleanInfo: Option[Int]): Boolean = { + tweetBooleanInfo.isDefined + } + private def isNotReply(tweetBooleanInfo: Option[Int]): Boolean = { + val isReply = isFeatureSet(IsReply, tweetBooleanInfo.getOrElse(0)) + !isReply + } + + private def isInNetwork(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = + candidate.features.getOrElse(FromInNetworkSourceFeature, false) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (kept, removed) = candidates.partition { candidate => + val tweetBooleanInfo = candidate.features.getOrElse(TweetBooleanInfoFeature, None) + isInNetwork(candidate) || (isVisible(tweetBooleanInfo) && isNotReply( + tweetBooleanInfo)) || shouldIgnore(candidate) + } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/AllowLowSignalUserGate.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/AllowLowSignalUserGate.scala new file mode 100644 index 000000000..eefd06284 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/AllowLowSignalUserGate.scala @@ -0,0 +1,29 @@ +package com.twitter.tweet_mixer.functional_component.gate + +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.functional_component.hydrator.SGSFollowedUsersFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableLowSignalUserCheck +import com.twitter.tweet_mixer.utils.SignalUtils + +/** + * Continue for low signal users who also follow < 25 users + * + * Check gate param last to only evaluate for eligible users and avoid experimental dilution. + */ +object AllowLowSignalUserGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("AllowLowSignalUser") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = + Stitch.value(evaluate(query)) + + def evaluate(query: PipelineQuery): Boolean = { + val followGraphSize = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty).size) + + SignalUtils.isLowSignalUser(query, followGraphSize) && + query.params(EnableLowSignalUserCheck) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/AllowNonEmptySearchHistoryUserGate.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/AllowNonEmptySearchHistoryUserGate.scala new file mode 100644 index 000000000..ade45ff50 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/AllowNonEmptySearchHistoryUserGate.scala @@ -0,0 +1,24 @@ +package com.twitter.tweet_mixer.functional_component.gate + +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableNonEmptySearchHistoryUserCheck + +/** + * Continue for users with non-empty search history + * + * Check gate param last to only evaluate for eligible users and avoid experimental dilution. + */ +case class AllowNonEmptySearchHistoryUserGate( + signalFn: PipelineQuery => Seq[String], +) extends Gate[PipelineQuery] { + override val identifier: GateIdentifier = GateIdentifier("AllowNonEmptySearchHistoryUser") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { + val queries: Seq[String] = signalFn(query) + val continue = queries.nonEmpty && query.params(EnableNonEmptySearchHistoryUserCheck) + Stitch.value(continue) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/BUILD.bazel new file mode 100644 index 000000000..3a1c47c82 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/BUILD.bazel @@ -0,0 +1,13 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/DenyLowSignalUserGate.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/DenyLowSignalUserGate.scala new file mode 100644 index 000000000..c96e2943d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/DenyLowSignalUserGate.scala @@ -0,0 +1,27 @@ +package com.twitter.tweet_mixer.functional_component.gate + +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.functional_component.hydrator.SGSFollowedUsersFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableLowSignalUserCheck +import com.twitter.tweet_mixer.utils.SignalUtils + +/** + * Continue for all users except low signal users who also follow < 25 users + * + * Check gate param last to only evaluate for eligible users and avoid experimental dilution. + */ +object DenyLowSignalUserGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("DenyLowSignalUser") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = + Stitch.value(evaluate(query)) + + def evaluate(query: PipelineQuery): Boolean = { + val followSize = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty).size) + !(SignalUtils.isLowSignalUser(query, followSize) && query.params(EnableLowSignalUserCheck)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/MaxFollowersGate.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/MaxFollowersGate.scala new file mode 100644 index 000000000..ba5f874a2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/MaxFollowersGate.scala @@ -0,0 +1,26 @@ +package com.twitter.tweet_mixer.functional_component.gate + +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.functional_component.hydrator.UserFollowersCountFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableMaxFollowersGate +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MaxFollowersCountGateParam + +/** + * Continue for users who have < N followers + * + * Check the enabled param at the end to prevent experiment dilution + */ +object MaxFollowersGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("MaxFollowers") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = Stitch.value { + val followers = + query.features.flatMap(_.getOrElse(UserFollowersCountFeature, None)).getOrElse(0) + val maxEligible = query.params(MaxFollowersCountGateParam) + followers < maxEligible || !query.params(EnableMaxFollowersGate) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/MinTimeSinceLastRequestGate.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/MinTimeSinceLastRequestGate.scala new file mode 100644 index 000000000..5842ec018 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/MinTimeSinceLastRequestGate.scala @@ -0,0 +1,31 @@ +package com.twitter.tweet_mixer.functional_component.gate + +import com.twitter.tweet_mixer.functional_component.hydrator.LastNonPollingTimeFeature +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationMinHoursSinceLastRequestParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LastNonPollingTimeFeatureHydratorEnabled + +/** + * Gate continues if the amount of time passed since the previous request is greater + * than the configured amount or if the previous request time in not available + */ +object MinTimeSinceLastRequestGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("TimeSinceLastRequest") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = Stitch.value { + // Always continue if the LastNonPollingTimeFeatureHydrator is not enabled + if (!query.params(LastNonPollingTimeFeatureHydratorEnabled)) { + return Stitch.value(true) + } + val minTimeSinceLastRequest = query.params(ContentExplorationMinHoursSinceLastRequestParam) + query.features.exists { features => + features + .getOrElse(LastNonPollingTimeFeature, None) + .forall(lnpt => (query.queryTime - lnpt) > minTimeSinceLastRequest) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/ProbablisticPassGate.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/ProbablisticPassGate.scala new file mode 100644 index 000000000..9073cd5f6 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate/ProbablisticPassGate.scala @@ -0,0 +1,23 @@ +package com.twitter.tweet_mixer.functional_component.gate + +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationOnceInTimesShow +import scala.util.Random + +/** + * Gate continues probablistically based on the FS config. + * If FS returns 3, it should return true once in 3 times. + */ +object ProbablisticPassGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("ProbablisticPass") + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = Stitch.value { + val onceInTimesShow = query.params(ContentExplorationOnceInTimesShow) + val randomDouble = Random.nextDouble() + randomDouble < (1.0 / onceInTimesShow) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/BUILD.bazel new file mode 100644 index 000000000..89fe4015b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/BUILD.bazel @@ -0,0 +1,73 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/com/twitter/storehaus:core", + "3rdparty/jvm/org/scalanlp:breeze", + "abuse/detection/src/main/thrift/com/twitter/abuse/detection/scoring:thrift-scala", + "configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider", + "finatra/inject/inject-core/src/main/scala", + "geoduck/service/src/main/scala/com/twitter/geoduck/service/common/clientmodules", + "geoduck/util/country", + "haplolite-thrift/thrift/src/main/thrift:thrift-scala", + "hydra/common/libraries/src/main/scala/com/twitter/hydra/common/utils", + "hydra/embedding-generation/thrift/src/main/thrift/com/twitter/hydra/embedding_generation:thrift-scala", + "hydra/root/thrift/src/main/thrift:thrift-scala", + "interests-service/thrift/src/main/thrift:thrift-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "servo/repo", + "src/thrift/com/twitter/recos/signals:thrift-scala", + "src/thrift/com/twitter/timelines/control_ai:timeline-control-ai-thrift-scala", + "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", + "src/thrift/com/twitter/tweetypie:service-scala", + "src/thrift/com/twitter/user_session_store:thrift-scala", + "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", + "stitch/stitch-core:cache", + "stitch/stitch-gizmoduck", + "stitch/stitch-socialgraph", + "stitch/stitch-tweetypie", + "strato/config/columns/content_understanding:content_understanding-strato-client", + "strato/config/columns/content_understanding/api:api-strato-client", + "strato/config/columns/hss/user_scores/api:api-strato-client", + "strato/config/columns/hydra:hydra-strato-client", + "strato/config/columns/interests:interests-strato-client", + "strato/config/columns/recommendations/signals:signals-strato-client", + "strato/config/columns/recommendations/user-signal-service:user-signal-service-strato-client", + "strato/config/columns/searchai/storage:storage-strato-client", + "strato/config/columns/timelines/control-ai/storage:storage-strato-client", + "strato/config/columns/tweetypie/managed:managed-strato-client", + "strato/config/columns/unified-counter/service:service-strato-client", + "strato/config/columns/videoRecommendations/twitterClip:twitterClip-strato-client", + "strato/config/columns/viewcounts:viewcounts-strato-client", + "strato/config/src/thrift/com/twitter/strato/columns/hydra:hydra-scala", + "strato/config/src/thrift/com/twitter/strato/graphql/unified_counter:graphql-unified-counter-scala", + "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", + "timelineservice/common:model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/uss_service", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/imv_related_tweets/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + "tweetsource/common/src/main/thrift:thrift-scala", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + "user_session_store/src/main/scala/com/twitter/user_session_store", + "viewcounts/src/main/thrift/com/twitter/viewcounts:viewcounts-scala", + ], + exports = [ + "3rdparty/jvm/javax/inject:javax.inject", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", + "src/thrift/com/twitter/spam/rtf:safety-result-scala", + "src/thrift/com/twitter/tweetypie:service-scala", + "src/thrift/com/twitter/tweetypie:tweet-scala", + "stitch/stitch-core", + "stitch/stitch-tweetypie", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentEmbeddingQueryFeatureHydratorFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentEmbeddingQueryFeatureHydratorFactory.scala new file mode 100644 index 000000000..ed72c2138 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentEmbeddingQueryFeatureHydratorFactory.scala @@ -0,0 +1,74 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.searchai.storage.SearchTweetEmbeddingXaiApiV2MHClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +object ContentExplorationEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Map[Long, Seq[Double]]]] { + override def defaultValue: Option[Map[Long, Seq[Double]]] = None +} + +@Singleton +class ContentExplorationEmbeddingQueryFeatureHydratorFactory @Inject() ( + embeddingClientColumn: SearchTweetEmbeddingXaiApiV2MHClientColumn) { + + def build( + signalFn: PipelineQuery => Seq[Long] + ): ContentExplorationEmbeddingQueryFeatureHydrator = { + new ContentExplorationEmbeddingQueryFeatureHydrator( + embeddingClientColumn, + signalFn + ) + } +} + +class ContentExplorationEmbeddingQueryFeatureHydrator( + searchTweetEmbeddingXaiApiV2MHClientColumn: SearchTweetEmbeddingXaiApiV2MHClientColumn, + signalFn: PipelineQuery => Seq[Long]) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ContentExplorationEmbedding") + + override val features: Set[Feature[_, _]] = Set(ContentExplorationEmbeddingFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val tweetIds = signalFn(query) + getEmbedding(tweetIds) + .map { embedding => + new FeatureMapBuilder() + .add(ContentExplorationEmbeddingFeature, embedding) + .build() + } + } + + private def getEmbedding( + tweetIds: Seq[Long] + ): Stitch[Option[Map[Long, Seq[Double]]]] = { + // Parallel fetch the embeddings for each tweetId + Stitch + .traverse(tweetIds) { tweetId => + searchTweetEmbeddingXaiApiV2MHClientColumn.fetcher + .fetch((tweetId, "")).map { result => + result.v match { + case Some(tweetEmbedding) if tweetEmbedding.embedding1.isDefined => + tweetEmbedding.embedding1.map(embedding => tweetId -> embedding) + case other => + None + } + } + }.map { results => + val map = results.flatten.toMap + if (map.nonEmpty) Some(map) else None + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentExplorationDRUserEmbeddingQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentExplorationDRUserEmbeddingQueryFeatureHydrator.scala new file mode 100644 index 000000000..bd872fee3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentExplorationDRUserEmbeddingQueryFeatureHydrator.scala @@ -0,0 +1,97 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.hydra.embedding_generation.{thriftscala => eg} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.utils.PipelineFailureCategories.FailedEmbeddingHydrationResponse +import com.twitter.tweet_mixer.utils.PipelineFailureCategories.InvalidEmbeddingHydrationResponse +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationDRUserEmbeddingModelName +import javax.inject.Inject +import javax.inject.Singleton + +object ContentExplorationDRUserEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Seq[Int]]] { + override def defaultValue: Option[Seq[Int]] = None +} + +@Singleton +class ContentExplorationDRUserEmbeddingQueryFeatureHydrator @Inject() ( + egsClient: eg.EmbeddingGenerationService.MethodPerEndpoint) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ContentExplorationDRUserEmbedding") + + override val features: Set[Feature[_, _]] = Set(ContentExplorationDRUserEmbeddingFeature) + + private val InvalidResponseException = Stitch.exception( + PipelineFailure(InvalidEmbeddingHydrationResponse, "Invalid embedding hydration response")) + + private val FailedResponseException = Stitch.exception( + PipelineFailure(FailedEmbeddingHydrationResponse, "Failed embedding hydration response")) + + /** + * The thrift response has been designed to support multiple embeddings for different model configs. For now + * we only expect to use a single config, so we will take the first one and ignore anything else in the response + */ + private object FirstResultInResponse { + def unapply(uer: eg.UserEmbeddingsResponse): Option[eg.UserEmbeddingsResult] = + uer.results.flatMap(_.values.headOption) + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + getEmbedding( + query.getRequiredUserId, + query.clientContext, + query.params(ContentExplorationDRUserEmbeddingModelName)) + .map { embedding => + new FeatureMapBuilder() + .add(ContentExplorationDRUserEmbeddingFeature, embedding) + .build() + } + } + + private def getEmbedding( + userId: Long, + clientContext: ClientContext, + modelName: String + ): Stitch[Option[Seq[Int]]] = { + val egsQuery = eg.EmbeddingsGenerationRequest( + clientContext = ClientContextMarshaller(clientContext), + product = eg.Product.UserEmbeddings, + productContext = Some( + eg.ProductContext.UserEmbeddingsContext( + eg.UserEmbeddingsContext(userIds = Seq(userId), modelNames = Some(Seq(modelName))))) + ) + + Stitch.callFuture(egsClient.generateEmbeddings(egsQuery)).flatMap { + case eg.EmbeddingsGenerationResponse + .UserEmbeddingsResponse(FirstResultInResponse(userEmbeddingsResult)) => + userEmbeddingsResult match { + case eg.UserEmbeddingsResult.UserEmbeddings(embeddings) => + // We expect the embeddings to be keyed on the user id that we passed + embeddings.embeddingsByUserId.flatMap(_.get(userId)) match { + case Some(embeddings) => + Stitch.value(Some(embeddings)) + case _ => Stitch.value(None) + } + case eg.UserEmbeddingsResult.ValidationError(error) => + Stitch.exception( + PipelineFailure( + InvalidEmbeddingHydrationResponse, + error.msg.getOrElse("Unknown validation error in EmbeddingsGenerationService"))) + case _ => InvalidResponseException + } + case _ => FailedResponseException + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentMediaEmbeddingQueryFeatureHydratorFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentMediaEmbeddingQueryFeatureHydratorFactory.scala new file mode 100644 index 000000000..ff278c1b1 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ContentMediaEmbeddingQueryFeatureHydratorFactory.scala @@ -0,0 +1,71 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.searchai.storage.PostAnnotationEmbeddingMHClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +object ContentMediaEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Map[Long, Seq[Double]]]] { + override def defaultValue: Option[Map[Long, Seq[Double]]] = None +} + +@Singleton +class ContentMediaEmbeddingQueryFeatureHydratorFactory @Inject() ( + embeddingClientColumn: PostAnnotationEmbeddingMHClientColumn) { + + def build( + signalFn: PipelineQuery => Seq[Long] + ): ContentMediaEmbeddingQueryFeatureHydrator = { + new ContentMediaEmbeddingQueryFeatureHydrator( + embeddingClientColumn, + signalFn + ) + } +} + +class ContentMediaEmbeddingQueryFeatureHydrator( + postAnnotationEmbeddingMHClientColumn: PostAnnotationEmbeddingMHClientColumn, + signalFn: PipelineQuery => Seq[Long]) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ContentMediaEmbedding") + + override val features: Set[Feature[_, _]] = Set(ContentMediaEmbeddingFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val tweetIds = signalFn(query) + getEmbedding(tweetIds) + .map { embedding => + new FeatureMapBuilder() + .add(ContentMediaEmbeddingFeature, embedding) + .build() + } + } + + private def getEmbedding( + tweetIds: Seq[Long] + ): Stitch[Option[Map[Long, Seq[Double]]]] = { + Stitch + .traverse(tweetIds) { tweetId => + postAnnotationEmbeddingMHClientColumn.fetcher + .fetch((tweetId, "")).map { result => + result.v match { + case Some(tweetEmbedding) => Some(tweetId -> tweetEmbedding.embedding) + case _ => None + } + } + }.map { results => + val map = results.flatten.toMap + if (map.nonEmpty) Some(map) else None + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ControlAiTopicEmbeddingQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ControlAiTopicEmbeddingQueryFeatureHydrator.scala new file mode 100644 index 000000000..976a00345 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ControlAiTopicEmbeddingQueryFeatureHydrator.scala @@ -0,0 +1,64 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.strato.generated.client.timelines.control_ai.storage.TopicEmbeddingMhClientColumn +import com.twitter.strato.generated.client.timelines.control_ai.storage.UserControlMhClientColumn +import com.twitter.stitch.Stitch +import com.twitter.timelines.control_ai.control.{thriftscala => ci} +import javax.inject.Inject +import javax.inject.Singleton + +object ControlAiTopicEmbeddings + extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Seq[Double]]] { + override def defaultValue: Seq[Seq[Double]] = Seq.empty +} + +@Singleton +class ControlAiTopicEmbeddingQueryFeatureHydrator @Inject() ( + userControlMhStratoColumn: UserControlMhClientColumn, + topicEmbeddingMhStratoColumn: TopicEmbeddingMhClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ControlAiTopicEmbedding") + + override val features: Set[Feature[_, _]] = Set(ControlAiTopicEmbeddings) + + private val validActions: Set[ci.ActionType] = Set(ci.ActionType.More, ci.ActionType.Only) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + + userControlMhStratoColumn.fetcher.fetch(query.getRequiredUserId).flatMap { result => + val topics: Seq[String] = result.v + .map(_.actions).map { + _.flatMap { action => + if (validActions.contains(action.actionType) && action.condition.postTopic.nonEmpty) + Some(action.condition.postTopic.get) + else + None + } + }.getOrElse(Seq.empty) + + val embeddingsStitch: Stitch[Seq[Seq[Double]]] = Stitch + .collect { + topics.map { topic => + topicEmbeddingMhStratoColumn.fetcher.fetch(topic).map { result => + result.v.map(_.embedding) + } + } + }.map(_.flatMap(_.toList)) + + embeddingsStitch.map { embeddings => + new FeatureMapBuilder() + .add(ControlAiTopicEmbeddings, embeddings) + .build() + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.scala new file mode 100644 index 000000000..35e7a04f8 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.scala @@ -0,0 +1,179 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.USSFeatures +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalI2iEmbModelName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetEmbeddingRandomSize +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetEmbeddingTimeDecay +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetEmbeddingSeedMaxAgeInDays +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetEmbeddingEngagementBoostWeight +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetEmbeddingTimeOfDayBoostWeight +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetEmbeddingDayOfWeekBoostWeight +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalTweetTweetEmbeddingMinPriority +import com.twitter.strato.generated.client.hydra.CachedEgsTweetEmbeddingClientColumn +import com.twitter.strato.columns.hydra.thriftscala.EmbeddingGenerationServiceParamsWithKey +import com.twitter.strato.columns.hydra.thriftscala.EmbeddingGenerationServiceParams +import com.twitter.util.Time +import com.twitter.util.Duration +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Random +import java.time.Instant +import java.time.ZoneId +import java.time.ZonedDateTime + +object DeepRetrievalTweetEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Map[Long, Seq[Int]]]] { + override def defaultValue: Option[Map[Long, Seq[Int]]] = None +} + +@Singleton +class DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory @Inject() ( + cachedEgsTweetEmbeddingClientColumn: CachedEgsTweetEmbeddingClientColumn) { + + def build( + signalFn: PipelineQuery => Seq[Long] + ): DeepRetrievalTweetEmbeddingQueryFeatureHydrator = { + new DeepRetrievalTweetEmbeddingQueryFeatureHydrator( + cachedEgsTweetEmbeddingClientColumn, + signalFn + ) + } +} + +class DeepRetrievalTweetEmbeddingQueryFeatureHydrator( + cachedEgsTweetEmbeddingClientColumn: CachedEgsTweetEmbeddingClientColumn, + signalFn: PipelineQuery => Seq[Long]) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("DeepRetrievalTweetEmbedding") + + override val features: Set[Feature[_, _]] = Set(DeepRetrievalTweetEmbeddingFeature) + + def sampleTweetIds( + ids: Seq[Long], + query: PipelineQuery + ): Seq[Long] = { + val n = query.params(DeepRetrievalTweetTweetEmbeddingRandomSize) + val timeDecay = query.params(DeepRetrievalTweetTweetEmbeddingTimeDecay) + val engagementBoostWeight = query.params(DeepRetrievalTweetTweetEmbeddingEngagementBoostWeight) + val timeOfDayBoostWeight = query.params(DeepRetrievalTweetTweetEmbeddingTimeOfDayBoostWeight) + val dayOfWeekBoostWeight = query.params(DeepRetrievalTweetTweetEmbeddingDayOfWeekBoostWeight) + val maxAge = query.params(DeepRetrievalTweetTweetEmbeddingSeedMaxAgeInDays) + val minPriority = query.params(DeepRetrievalTweetTweetEmbeddingMinPriority) + + // If n is non-positive, return all ids. + if (n <= 0) return ids + + // If timeDecay is not set or is out of range, use random sampling. + if (timeDecay <= 0.0 || timeDecay >= 1.0) { + return Random.shuffle(ids).take(n) + } + + // Filter out ids based on timestamp. + val idTosignals = USSFeatures.getSignalsWithInfo(query, USSFeatures.TweetFeatures).map { + case (id, signalInfos) => + val priority = USSFeatures.getPriority(signalInfos) + val mostRecentTimeStamp = + signalInfos + .flatMap(_.sourceEventTime) + .foldLeft(Time.Zero)(_ max _) + (id, (priority, mostRecentTimeStamp)) + } + val filteredIds = ids.filter { id => + idTosignals.get(id) match { + case Some((priority, timestamp)) => + val passesAgeFilter = maxAge <= Duration.fromDays(0) || (Time.now - timestamp <= maxAge) + val passesPriorityFilter = minPriority < 0 || priority >= minPriority + passesAgeFilter && passesPriorityFilter + case None => false + } + } + + // Select ids based on priority and time. + val weightedIds = filteredIds.map { id => + idTosignals.get(id) match { + case Some((priority, timestamp)) => + val matchTimeOfDay = isMatchTimeOfDay(timestamp) + val matchDayOfWeek = isMatchDayOfWeek(timestamp) + val weight = math.pow(timeDecay, (Time.now - timestamp).inDays) * + math.min(priority * engagementBoostWeight, 1.0) * + (if (matchTimeOfDay) timeOfDayBoostWeight else 1.0) * + (if (matchDayOfWeek) dayOfWeekBoostWeight else 1.0) + (id, weight) + case None => + (id, 0.0) + } + } + weightedIds.sortBy(_._2).take(n).map(_._1) + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val tweetIds = signalFn(query) + val sampledTweetIds = sampleTweetIds(tweetIds, query) + getEmbedding(sampledTweetIds, query.clientContext, query.params(DeepRetrievalI2iEmbModelName)) + .map { embedding => + new FeatureMapBuilder() + .add(DeepRetrievalTweetEmbeddingFeature, embedding) + .build() + } + } + + private def getEmbedding( + tweetIds: Seq[Long], + clientContext: ClientContext, + modelName: String + ): Stitch[Option[Map[Long, Seq[Int]]]] = { + val egsParams = EmbeddingGenerationServiceParams( + deepRetrievalModels = Some(true), + modelNames = Some(Seq(modelName)) + ) + + // Parallel fetch the embeddings for each tweetId + Stitch + .traverse(tweetIds) { tweetId => + val egsKey = EmbeddingGenerationServiceParamsWithKey( + tweetId = tweetId, + params = egsParams, + ) + cachedEgsTweetEmbeddingClientColumn.fetcher.fetch(egsKey).flatMap { result => + result.v match { + case Some(embeddingsByModel: Map[String, Seq[Int]]) => + Stitch.value(embeddingsByModel.get(modelName).map(embedding => tweetId -> embedding)) + case other => + Stitch.value(None) + } + } + }.map { results => + val map = results.flatten.toMap + if (map.nonEmpty) Some(map) else None + } + } + + def isMatchTimeOfDay(time: Time, windowHours: Int = 2): Boolean = { + val millisInDay = 24 * 60 * 60 * 1000L + val now = Time.now + val nowMillisOfDay = now.inMilliseconds % millisInDay + val timeMillisOfDay = time.inMilliseconds % millisInDay + val diff = math.abs(nowMillisOfDay - timeMillisOfDay) + val minDiff = math.min(diff, millisInDay - diff) // handle wrap-around at midnight + minDiff <= windowHours * 60 * 60 * 1000L + } + + def isMatchDayOfWeek(time: Time): Boolean = { + val zdt = + ZonedDateTime.ofInstant(Instant.ofEpochMilli(time.inMilliseconds), ZoneId.systemDefault()) + val nowZdt = + ZonedDateTime.ofInstant(Instant.ofEpochMilli(Time.now.inMilliseconds), ZoneId.systemDefault()) + zdt.getDayOfWeek == nowZdt.getDayOfWeek + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/DeepRetrievalUserEmbeddingQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/DeepRetrievalUserEmbeddingQueryFeatureHydrator.scala new file mode 100644 index 000000000..bec3b5c2b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/DeepRetrievalUserEmbeddingQueryFeatureHydrator.scala @@ -0,0 +1,121 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.hydra.embedding_generation.{thriftscala => eg} +import com.twitter.hydra.common.utils.{Utils => HydraUtils} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalAddUserEmbeddingGaussianNoise +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalUserEmbeddingGaussianNoiseParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalModelName +import com.twitter.tweet_mixer.utils.PipelineFailureCategories.FailedEmbeddingHydrationResponse +import com.twitter.tweet_mixer.utils.PipelineFailureCategories.InvalidEmbeddingHydrationResponse +import javax.inject.Inject +import javax.inject.Singleton +import scala.util.Random +import scala.jdk.CollectionConverters._ + +object DeepRetrievalUserEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Seq[Int]]] { + override def defaultValue: Option[Seq[Int]] = None +} + +@Singleton +class DeepRetrievalUserEmbeddingQueryFeatureHydrator @Inject() ( + egsClient: eg.EmbeddingGenerationService.MethodPerEndpoint) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("DeepRetrievalUserEmbedding") + + override val features: Set[Feature[_, _]] = Set(DeepRetrievalUserEmbeddingFeature) + + private val InvalidResponseException = Stitch.exception( + PipelineFailure(InvalidEmbeddingHydrationResponse, "Invalid embedding hydration response")) + + private val FailedResponseException = Stitch.exception( + PipelineFailure(FailedEmbeddingHydrationResponse, "Failed embedding hydration response")) + + /** + * The thrift response has been designed to support multiple embeddings for different model configs. For now + * we only expect to use a single config, so we will take the first one and ignore anything else in the response + */ + private object FirstResultInResponse { + def unapply(uer: eg.UserEmbeddingsResponse): Option[eg.UserEmbeddingsResult] = + uer.results.flatMap(_.values.headOption) + } + + private def addGaussianNoise( + query: PipelineQuery, + embedding: Option[Seq[Int]] + ): Option[Seq[Int]] = { + if (query.params(DeepRetrievalAddUserEmbeddingGaussianNoise)) { + val rng = new Random() + val noiseMultiplier = query.params.getDouble(DeepRetrievalUserEmbeddingGaussianNoiseParam) + embedding.map { vec => + val floatVector = HydraUtils.intBitsSeqToFloatSeq(vec) + val noisyFloatVector = floatVector.map { value => + value + (rng.nextGaussian() * noiseMultiplier).toFloat + } + HydraUtils + .floatToIntegerJList(noisyFloatVector.map(java.lang.Float.valueOf).asJava).asScala.map( + _.intValue) + } + } else { + embedding + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + getEmbedding(query.getRequiredUserId, query.clientContext, query.params(DeepRetrievalModelName)) + .map { embedding => + val userEmbedding = addGaussianNoise(query, embedding) + new FeatureMapBuilder() + .add(DeepRetrievalUserEmbeddingFeature, userEmbedding) + .build() + } + } + + private def getEmbedding( + userId: Long, + clientContext: ClientContext, + modelName: String + ): Stitch[Option[Seq[Int]]] = { + val egsQuery = eg.EmbeddingsGenerationRequest( + clientContext = ClientContextMarshaller(clientContext), + product = eg.Product.UserEmbeddings, + productContext = Some( + eg.ProductContext.UserEmbeddingsContext( + eg.UserEmbeddingsContext(userIds = Seq(userId), modelNames = Some(Seq(modelName))))) + ) + + Stitch.callFuture(egsClient.generateEmbeddings(egsQuery)).flatMap { + case eg.EmbeddingsGenerationResponse + .UserEmbeddingsResponse(FirstResultInResponse(userEmbeddingsResult)) => + userEmbeddingsResult match { + case eg.UserEmbeddingsResult.UserEmbeddings(embeddings) => + // We expect the embeddings to be keyed on the user id that we passed + embeddings.embeddingsByUserId.flatMap(_.get(userId)) match { + case Some(embeddings) => + Stitch.value(Some(embeddings)) + case _ => Stitch.value(None) + } + case eg.UserEmbeddingsResult.ValidationError(error) => + Stitch.exception( + PipelineFailure( + InvalidEmbeddingHydrationResponse, + error.msg.getOrElse("Unknown validation error in EmbeddingsGenerationService"))) + case _ => InvalidResponseException + } + case _ => FailedResponseException + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/EvergreenDRUserEmbeddingQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/EvergreenDRUserEmbeddingQueryFeatureHydrator.scala new file mode 100644 index 000000000..828396ca7 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/EvergreenDRUserEmbeddingQueryFeatureHydrator.scala @@ -0,0 +1,97 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.hydra.embedding_generation.{thriftscala => eg} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.utils.PipelineFailureCategories.FailedEmbeddingHydrationResponse +import com.twitter.tweet_mixer.utils.PipelineFailureCategories.InvalidEmbeddingHydrationResponse +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenDRUserEmbeddingModelName +import javax.inject.Inject +import javax.inject.Singleton + +object EvergreenDRUserEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Seq[Int]]] { + override def defaultValue: Option[Seq[Int]] = None +} + +@Singleton +class EvergreenDRUserEmbeddingQueryFeatureHydrator @Inject() ( + egsClient: eg.EmbeddingGenerationService.MethodPerEndpoint) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("EvergreenDRUserEmbedding") + + override val features: Set[Feature[_, _]] = Set(EvergreenDRUserEmbeddingFeature) + + private val InvalidResponseException = Stitch.exception( + PipelineFailure(InvalidEmbeddingHydrationResponse, "Invalid embedding hydration response")) + + private val FailedResponseException = Stitch.exception( + PipelineFailure(FailedEmbeddingHydrationResponse, "Failed embedding hydration response")) + + /** + * The thrift response has been designed to support multiple embeddings for different model configs. For now + * we only expect to use a single config, so we will take the first one and ignore anything else in the response + */ + private object FirstResultInResponse { + def unapply(uer: eg.UserEmbeddingsResponse): Option[eg.UserEmbeddingsResult] = + uer.results.flatMap(_.values.headOption) + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + getEmbedding( + query.getRequiredUserId, + query.clientContext, + query.params(EvergreenDRUserEmbeddingModelName)) + .map { embedding => + new FeatureMapBuilder() + .add(EvergreenDRUserEmbeddingFeature, embedding) + .build() + } + } + + private def getEmbedding( + userId: Long, + clientContext: ClientContext, + modelName: String + ): Stitch[Option[Seq[Int]]] = { + val egsQuery = eg.EmbeddingsGenerationRequest( + clientContext = ClientContextMarshaller(clientContext), + product = eg.Product.UserEmbeddings, + productContext = Some( + eg.ProductContext.UserEmbeddingsContext( + eg.UserEmbeddingsContext(userIds = Seq(userId), modelNames = Some(Seq(modelName))))) + ) + + Stitch.callFuture(egsClient.generateEmbeddings(egsQuery)).flatMap { + case eg.EmbeddingsGenerationResponse + .UserEmbeddingsResponse(FirstResultInResponse(userEmbeddingsResult)) => + userEmbeddingsResult match { + case eg.UserEmbeddingsResult.UserEmbeddings(embeddings) => + // We expect the embeddings to be keyed on the user id that we passed + embeddings.embeddingsByUserId.flatMap(_.get(userId)) match { + case Some(embeddings) => + Stitch.value(Some(embeddings)) + case _ => Stitch.value(None) + } + case eg.UserEmbeddingsResult.ValidationError(error) => + Stitch.exception( + PipelineFailure( + InvalidEmbeddingHydrationResponse, + error.msg.getOrElse("Unknown validation error in EmbeddingsGenerationService"))) + case _ => InvalidResponseException + } + case _ => FailedResponseException + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/FeedbackHistoryQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/FeedbackHistoryQueryFeatureHydrator.scala new file mode 100644 index 000000000..5fbc61e60 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/FeedbackHistoryQueryFeatureHydrator.scala @@ -0,0 +1,58 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClient +import com.twitter.timelineservice.model.FeedbackEntry +import com.twitter.tweet_mixer.feature.EntityTypes.TweetId +import com.twitter.tweet_mixer.feature.EntityTypes.UserId +import com.twitter.tweet_mixer.feature.USSFeature +import com.twitter.usersignalservice.{thriftscala => uss} +import javax.inject.Inject +import javax.inject.Singleton + +object TweetShowMore extends USSFeature[TweetId] + +object AccountShowMore extends USSFeature[UserId] + +@Singleton +case class FeedbackHistoryQueryFeatureHydrator @Inject() ( + feedbackHistoryClient: FeedbackHistoryManhattanClient) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory") + + override val features: Set[Feature[_, _]] = Set(TweetShowMore, AccountShowMore) + + override def hydrate( + query: PipelineQuery + ): Stitch[FeatureMap] = Stitch + .callFuture(feedbackHistoryClient.get(query.getRequiredUserId)) + .map { feedbackHistory => + val tweetShowMoreSignals = + feedbackHistoryToSignal(feedbackHistory, uss.SignalType.TweetSeeMore) + val accountShowMoreSignals = + feedbackHistoryToSignal(feedbackHistory, uss.SignalType.AccountSeeMore) + + FeatureMap( + TweetShowMore, + USSQueryFeatureHydrator.getSignalMap[TweetId](tweetShowMoreSignals), + AccountShowMore, + USSQueryFeatureHydrator.getSignalMap[UserId](accountShowMoreSignals) + ) + } + + private def feedbackHistoryToSignal( + feedbackHistory: Seq[FeedbackEntry], + signalType: uss.SignalType + ): Seq[uss.Signal] = feedbackHistory.map { feedbackEntry => + uss.Signal( + signalType = signalType, + timestamp = feedbackEntry.timestamp.inMilliseconds + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GizmoduckQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GizmoduckQueryFeatureHydrator.scala new file mode 100644 index 000000000..12d124f59 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GizmoduckQueryFeatureHydrator.scala @@ -0,0 +1,45 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.gizmoduck.{thriftscala => gt} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.stitch.gizmoduck.Gizmoduck +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableGizmoduckQueryFeatureHydrator +import javax.inject.Inject +import javax.inject.Singleton + +object UserFollowersCountFeature extends Feature[PipelineQuery, Option[Int]] + +@Singleton +case class GizmoduckQueryFeatureHydrator @Inject() (gizmoduck: Gizmoduck) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Gizmoduck") + + override val features: Set[Feature[_, _]] = Set(UserFollowersCountFeature) + + private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Counts) + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableGizmoduckQueryFeatureHydrator) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + gizmoduck + .getUserById( + userId = userId, + queryFields = queryFields, + context = gt.LookupContext(forUserId = Some(userId)) + ).map { user => + FeatureMap(UserFollowersCountFeature, user.counts.map(_.followers.toInt)) + }.rescue { + case _ => Stitch.value(FeatureMap(UserFollowersCountFeature, None)) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokBooleanFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokBooleanFeatureHydrator.scala new file mode 100644 index 000000000..97e6a911f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokBooleanFeatureHydrator.scala @@ -0,0 +1,72 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.content_understanding.api.PostAnnotationsOnTweetClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +object GrokIsNsfwFeature extends Feature[TweetCandidate, Option[Boolean]] + +object GrokIsSoftNsfwFeature extends Feature[TweetCandidate, Option[Boolean]] + +@Singleton +class GrokBooleanFeatureHydrator @Inject() ( + postAnnotationsOnTweetClientColumn: PostAnnotationsOnTweetClientColumn) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GrokBoolean") + + override val features: Set[Feature[_, _]] = + Set(GrokIsNsfwFeature, GrokIsSoftNsfwFeature) + + private val fetcher = postAnnotationsOnTweetClientColumn.fetcher + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + + if (candidates.isEmpty) { + Stitch.value(Seq.empty) + } else { + val tweetIds = candidates.map(_.candidate.id) + + val stitchedResults = tweetIds.map { tweetId => + fetcher + .fetch(tweetId).map { response => + val resultOpt = response.v + + val (isNsfw, isSoftNsfw) = resultOpt match { + case Some(result) => + val boolMetadata = result.annotations.tweetBoolMetadata + val nsfw = boolMetadata.flatMap(_.isNsfw) + val softNsfw = boolMetadata.flatMap(_.isSoftNsfw) + (nsfw, softNsfw) + case None => + (Some(false), Some(false)) + } + FeatureMapBuilder() + .add(GrokIsNsfwFeature, isNsfw) + .add(GrokIsSoftNsfwFeature, isSoftNsfw) + .build() + }.handle { + case _ => + FeatureMapBuilder() + .add(GrokIsNsfwFeature, Some(false)) + .add(GrokIsSoftNsfwFeature, Some(false)) + .build() + } + } + + Stitch.collect(stitchedResults) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokCategoriesFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokCategoriesFeatureHydrator.scala new file mode 100644 index 000000000..c6e762d95 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokCategoriesFeatureHydrator.scala @@ -0,0 +1,30 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.interests.FetchXAICategoriesClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +object GrokCategoriesFeature extends Feature[PipelineQuery, Option[Seq[Long]]] + +@Singleton +class GrokCategoriesQueryFeatureHydrator @Inject() ( + fetchXAICategoriesClientColumn: FetchXAICategoriesClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GrokCategories") + + override val features: Set[Feature[_, _]] = Set(GrokCategoriesFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + fetchXAICategoriesClientColumn.fetcher.fetch().map { response => + val categories = response.v.map { s => s.map(_._1) } + FeatureMap(GrokCategoriesFeature, categories) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokFilterFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokFilterFeatureHydrator.scala new file mode 100644 index 000000000..1eb2f47d6 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/GrokFilterFeatureHydrator.scala @@ -0,0 +1,101 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.stitch.cache.AsyncValueCache +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import com.twitter.strato.generated.client.content_understanding.ColdStartPostsMetadataMhClientColumn +import com.twitter.strato.columns.content_understanding.content_exploration.thriftscala.ColdStartPostStatus +import javax.inject.Inject +import javax.inject.Singleton + +object GrokFilterFeature extends Feature[TweetCandidate, Boolean] + +@Singleton +class GrokFilterFeatureHydrator @Inject() ( + coldStartPostsMetadataMhClientColumn: ColdStartPostsMetadataMhClientColumn, + statsReceiver: StatsReceiver, + memcache: MemcacheStitchClient, + inMemoryCache: AsyncValueCache[java.lang.Long, Boolean]) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GrokFilter") + + override val features: Set[Feature[_, _]] = Set(GrokFilterFeature) + + private val grokFilterTrueStat = statsReceiver.counter("GrokFilterTrueCount") + private val grokFilterFalseStat = statsReceiver.counter("GrokFilterFalseCount") + + private val inMemoryCacheRequestCounter = statsReceiver.counter("InMemoryCacheRequests") + private val memcacheRequestsCounter = statsReceiver.counter("MemcacheMisses") + private val manhattanRequestCounter = statsReceiver.counter("ManhattanRequests") + + private val TTLSeconds = Utils.randomizedTTL(10 * 60) + private val DefaultFeatureMap = FeatureMap(GrokFilterFeature, false) + + private def getCacheKey(tweetId: Long): String = "GrokFilter:" + tweetId.toString + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val featureMaps = candidates.map { candidate => + val postId = candidate.candidate.id + inMemoryCacheRequestCounter.incr() + inMemoryCache + .get(postId).getOrElse(fetchFromMemcache(postId)).map { filter => + if (filter) grokFilterTrueStat.incr() else grokFilterFalseStat.incr() + FeatureMap(GrokFilterFeature, filter) + } + .handle { case _ => DefaultFeatureMap } + } + + Stitch.collect(featureMaps) + } + + private def fetchFromMemcache(tweetId: Long): Stitch[Boolean] = { + memcacheRequestsCounter.incr() + + memcache + .get(getCacheKey(tweetId)) + .flatMap { + case Some(value) => + val filter = Transformers.deserializeFilterBoolean(value) + Stitch.value(filter).applyEffect { result => + Stitch.async { inMemoryCache.set(tweetId, Stitch.value(result)) } + } + case None => + fetchFromManhattan(tweetId) + .map { result => + result match { + case Some(status) if status != ColdStartPostStatus.Tier1Ineligible => false + case _ => true + } + }.applyEffect { filter => + Stitch.async { + val buf = Transformers.serializeFilterBoolean(filter) + val memCacheSet = memcache.set(getCacheKey(tweetId), buf, TTLSeconds) + val inMemoryCacheSet = inMemoryCache.set(tweetId, Stitch.value(filter)) + Stitch.join(inMemoryCacheSet, memCacheSet).map { _ => () } + } + } + } + } + + private def fetchFromManhattan(postId: Long): Stitch[Option[ColdStartPostStatus]] = { + manhattanRequestCounter.incr() + coldStartPostsMetadataMhClientColumn.fetcher + .fetch(postId) + .map { response => response.v.flatMap(_.status) } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HaploliteQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HaploliteQueryFeatureHydrator.scala new file mode 100644 index 000000000..97b849b23 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HaploliteQueryFeatureHydrator.scala @@ -0,0 +1,91 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.haplolite.{thriftscala => t} +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.stitch.Stitch +import com.twitter.timelines.util.ByteBufferBuilder +import com.twitter.timelines.util.SnowflakeSortIndexHelper +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.HaploliteTweetsBasedEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MaxHaploliteTweetsParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MaxTweetAgeHoursParam +import com.twitter.util.Future +import com.twitter.util.Time +import java.nio.ByteBuffer +import javax.inject.Inject +import javax.inject.Singleton + +object HaploliteFeature extends Feature[PipelineQuery, Seq[ByteBuffer]] +object HaploMissFeature extends Feature[PipelineQuery, Boolean] + +@Singleton +case class HaploliteQueryFeatureHydrator @Inject() ( + haploliteClient: t.Haplolite.MethodPerEndpoint) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Haplolite") + + override val features: Set[Feature[_, _]] = Set(HaploliteFeature, HaploMissFeature) + + private def idToBuf(id: Long) = { + ByteBufferBuilder(8) { _.putLong(id) } + } + + override def onlyIf(query: PipelineQuery): Boolean = query.params(HaploliteTweetsBasedEnabled) + + private val defaultFeatureMap = FeatureMap(HaploliteFeature, Seq.empty, HaploMissFeature, false) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val haploliteRequest = createHaploliteRequest(query) + val haploliteResultFut = haploliteClient.get(Seq(haploliteRequest)) + val haploliteEntriesFut = + haploliteResultFut + .map { haploliteResults => + haploliteResults.list.flatMap { getResult => + getResult.timeline.map(_.entries).getOrElse(Seq.empty) + } + } + val haploliteMissFut = haploliteResultFut + .map { haploliteResults => + haploliteResults.list.exists { getResult => + getResult.timeline.map(_.state == t.ResultState.Miss).getOrElse(false) + } + } + Stitch + .callFuture { + Future.join(haploliteEntriesFut, haploliteMissFut).map { + case (haploliteEntries, haploliteMiss) => + FeatureMap(HaploliteFeature, haploliteEntries, HaploMissFeature, haploliteMiss) + } + }.handle { + case _ => defaultFeatureMap + } + } + + private def createHaploliteRequest(query: PipelineQuery): t.GetRequest = { + val userId = query.getRequiredUserId + + val duration = query.params(MaxTweetAgeHoursParam) + val sinceTime: Time = duration.ago + val fromTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(sinceTime) + + val bulkKeys = t.BulkKeys(t.KeyNamespace.Home.value, Seq(userId)) + val haploliteCursor = + t.Cursor(comparableEntry = t.ComparableEntry(idToBuf(fromTweetIdExclusive)), isForward = true) + t.GetRequest( + bulkKeys = Seq(bulkKeys), + timelineRequest = Some( + t.TimelineRequest( + dedupeSecondary = true, + maxCount = query.params(MaxHaploliteTweetsParam), + cursor = Some(haploliteCursor) + ) + ) + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HighQualitySourceSignalQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HighQualitySourceSignalQueryFeatureHydrator.scala new file mode 100644 index 000000000..424cf1e18 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HighQualitySourceSignalQueryFeatureHydrator.scala @@ -0,0 +1,173 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.recos.signals.thriftscala.SourceSignal +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.recommendations.signals.RetrievalSignalsOnUserClientColumn +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.Params +import com.twitter.tweet_mixer.feature._ +import com.twitter.tweet_mixer.feature.EntityTypes._ +import com.twitter.tweet_mixer.param.HighQualitySourceSignalParams._ +import com.twitter.tweet_mixer.param.USSParams.UnifiedMaxSourceKeyNum +import com.twitter.usersignalservice.thriftscala.SignalEntity +import com.twitter.usersignalservice.thriftscala.SignalType +import com.twitter.util.Return +import com.twitter.util.Throw +import com.twitter.util.Time +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class HighQualitySourceSignalQueryFeatureHydrator @Inject() ( + retrievalSignalsClientColumn: RetrievalSignalsOnUserClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("HighQualitySourceSignal") + + override val features: Set[Feature[_, _]] = Set(HighQualitySourceTweetV2, HighQualitySourceUserV2) + + val fetcher: Fetcher[ + RetrievalSignalsOnUserClientColumn.Key, + Unit, + RetrievalSignalsOnUserClientColumn.Value + ] = + retrievalSignalsClientColumn.fetcher + + val dayMs = 24 * 60 * 60 * 1000L + val defaultTimestamp = 0L + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + fetcher.fetch(query.getRequiredUserId).map(_.v).transform { response => + val sourceSignalList: Seq[SourceSignal] = response match { + case Return(result) => result.map(_.sourceSignalList).getOrElse(Seq.empty) + case Throw(ex) => Seq.empty + case _ => Seq.empty + } + val featureMapBuilder = FeatureMapBuilder() + buildSignalFeature[TweetId]( + sourceSignalList, + SignalEntity.Tweet, + SignalType.HighQualitySourceTweet, + featureMapBuilder, + HighQualitySourceTweetV2, + query.params, + EnableHighQualitySourceTweetV2, + MaxHighQualitySourceSignalsV2 + ) + buildSignalFeature[UserId]( + sourceSignalList, + SignalEntity.User, + SignalType.HighQualitySourceUser, + featureMapBuilder, + HighQualitySourceUserV2, + query.params, + EnableHighQualitySourceUserV2, + MaxHighQualitySourceSignalsV2 + ) + Stitch.value(featureMapBuilder.build()) + } + } + + def buildSignalFeature[T]( + sourceSignalList: Seq[SourceSignal], + entityType: SignalEntity, + signalType: SignalType, + featureMapBuilder: FeatureMapBuilder, + featureType: USSFeature[T], + params: Params, + sourceSignalParam: FSParam[Boolean], + maxResultsParam: FSBoundedParam[Int] = UnifiedMaxSourceKeyNum + ): Unit = { + val decayRate = params(TimeDecayRate) + val enableTimeDecay = params(EnableTimeDecay) + val currentTime = Time.now.inMilliseconds + val signalMap: Map[T, Seq[SignalInfo]] = if (params(sourceSignalParam)) { + + val filterSourceSignals = sourceSignalList.filter(_.sourceSignalEntity.contains(entityType)) + val signals = filterSourceSignals + .map(signal => (signal, getScore(enableTimeDecay, decayRate, currentTime, signal, params))) + .filter(_._2 > 0) + .sortBy { + case (signal, score) => + (-score, getNegativeCount(signal), -getTimestamp(signal)) + } + .take(params(maxResultsParam)) + .map(_._1) + signals + .map { signal => + ( + signal.sourceSignalId.asInstanceOf[T], + SignalInfo( + signalEntity = entityType, + signalType = signalType, + sourceEventTime = Some(Time.fromMilliseconds(getTimestamp(signal))), + authorId = None + ) + ) + } + .groupBy(_._1).mapValues(_.map(_._2)) + } else { + Map.empty[T, Seq[SignalInfo]] + } + featureMapBuilder.add(featureType, signalMap) + } + + def getScore( + enableTimeDecay: Boolean, + decayRate: Double, + currentTime: Long, + signal: SourceSignal, + params: Params + ): Double = { + val timeDiffDays: Double = (currentTime - getTimestamp(signal)).toDouble / dayMs + if (enableTimeDecay) { + calculateScore(signal, params) * Math.exp(-decayRate * timeDiffDays) + } else { + calculateScore(signal, params) + } + } + + def getTimestamp(signal: SourceSignal): Long = { + signal.updatedAt.orElse(signal.createdAt).getOrElse(defaultTimestamp) + } + + // Computes the score for a SourceSignal using weights from HighQualitySourceSignalParams. + private def calculateScore( + signal: SourceSignal, + params: Params + ): Double = { + signal.bookmarkCount.map(_ * params(HighQualitySourceSignalBookmarkWeight)).getOrElse(0.0) + + signal.favCount.map(_ * params(HighQualitySourceSignalFavWeight)).getOrElse(0.0) + + signal.replyCount.map(_ * params(HighQualitySourceSignalReplyWeight)).getOrElse(0.0) + + signal.retweetCount.map(_ * params(HighQualitySourceSignalRetweetWeight)).getOrElse(0.0) + + signal.quoteCount.map(_ * params(HighQualitySourceSignalQuoteWeight)).getOrElse(0.0) + + signal.shareCount.map(_ * params(HighQualitySourceSignalShareWeight)).getOrElse(0.0) + + signal.videoQualityViewCount + .map(_ * params(HighQualitySourceSignalVideoQualityViewWeight)).getOrElse(0.0) + + signal.tweetDetailsClickCount + .map(_ * params(HighQualitySourceSignalTweetDetailsClickWeight)).getOrElse(0.0) + + signal.tweetDetailsImpressionCount + .map(_ * params(HighQualitySourceSignalTweetDetailsImpressionWeight)).getOrElse(0.0) + + signal.notInterestedCount + .map(_ * params(HighQualitySourceSignalNotInterestedWeight)).getOrElse(0.0) + + signal.blockCount.map(_ * params(HighQualitySourceSignalBlockWeight)).getOrElse(0.0) + + signal.muteCount.map(_ * params(HighQualitySourceSignalMuteWeight)).getOrElse(0.0) + + signal.reportCount.map(_ * params(HighQualitySourceSignalReportWeight)).getOrElse(0.0) + } + + def getNegativeCount( + sourceSignal: SourceSignal + ): Int = { + sourceSignal.notInterestedCount.getOrElse(0) + sourceSignal.blockCount.getOrElse( + 0) + sourceSignal.muteCount.getOrElse(0) + sourceSignal.reportCount.getOrElse(0) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HydraRankingPreparationQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HydraRankingPreparationQueryFeatureHydrator.scala new file mode 100644 index 000000000..3106cfe96 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/HydraRankingPreparationQueryFeatureHydrator.scala @@ -0,0 +1,96 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.hydra.root.thriftscala.HydraRootRecommendationResponse +import com.twitter.hydra.root.{thriftscala => t} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableHydraScoringSideEffect +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.HydraModelName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.HydraScoringPipelineEnabled +import javax.inject.Inject +import javax.inject.Singleton + +object HydraRankingPreparedFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Boolean] { + override def defaultValue: Boolean = false +} + +@Singleton +class HydraRankingPreparationQueryFeatureHydrator @Inject() ( + hydraRootService: t.HydraRoot.MethodPerEndpoint) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("HydraRootService") + + override val features: Set[Feature[_, _]] = Set(HydraRankingPreparedFeature) + + private val succeededFeatureMap = FeatureMapBuilder() + .add(HydraRankingPreparedFeature, true) + .build() + + private val failedFeatureMap = FeatureMapBuilder() + .add(HydraRankingPreparedFeature, false) + .build() + + override def onlyIf(query: PipelineQuery): Boolean = { + // Using same param as side effect for making sure, both ranking and preparation happens for same request + query.params(EnableHydraScoringSideEffect) || query.params(HydraScoringPipelineEnabled) + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + val clientContext = query.clientContext + val modelName = query.params(HydraModelName) + if (query.params(HydraScoringPipelineEnabled)) { + syncHydraCall(userId, clientContext, modelName) + } else if (query.params(EnableHydraScoringSideEffect)) { + asyncHydraCall(userId, clientContext, modelName) + } else { + Stitch.value(failedFeatureMap) // Should never reach here as onlyIf will gate + } + } + + private def asyncHydraCall( + userId: Long, + clientContext: ClientContext, + modelName: String + ): Stitch[FeatureMap] = Stitch + .value(succeededFeatureMap) + .applyEffect { _ => + Stitch.async { getHydraResponse(userId, clientContext, modelName) } + } + + private def syncHydraCall( + userId: Long, + clientContext: ClientContext, + modelName: String + ): Stitch[FeatureMap] = getHydraResponse(userId, clientContext, modelName).map { + case HydraRootRecommendationResponse.HydraRootTweetRankingPreparationResponse(_) => + succeededFeatureMap + case _ => failedFeatureMap + } + + private def getHydraResponse( + userId: Long, + clientContext: ClientContext, + modelName: String + ): Stitch[HydraRootRecommendationResponse] = { + val hydraRequest = t.HydraRootRequest( + clientContext = ClientContextMarshaller(clientContext), + product = t.Product.TweetRankingPreparation, + productContext = + Some(t.ProductContext.TweetRankingPreparation(t.TweetRankingPreparation(modelName))), + ) + + Stitch.callFuture(hydraRootService.getRecommendationResponse(hydraRequest)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala new file mode 100644 index 000000000..310774e8b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala @@ -0,0 +1,60 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.storehaus.Store +import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} +import com.twitter.tweet_mixer.model.ModuleNames.MemcachedImpressionBloomFilterStore +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableImpressionBloomFilterHydrator +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +object ImpressionBloomFilterFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, blm.ImpressionBloomFilterSeq] { + override def defaultValue: blm.ImpressionBloomFilterSeq = + blm.ImpressionBloomFilterSeq(Seq.empty) +} + +@Singleton +case class ImpressionBloomFilterQueryFeatureHydratorFactory @Inject() ( + @Named(MemcachedImpressionBloomFilterStore) bloomFilterClient: Store[ + blm.ImpressionBloomFilterKey, + blm.ImpressionBloomFilterSeq + ]) { + + def build[Query <: PipelineQuery]( + surfaceArea: blm.SurfaceArea + ): ImpressionBloomFilterQueryFeatureHydrator[Query] = + ImpressionBloomFilterQueryFeatureHydrator(bloomFilterClient, surfaceArea) +} + +case class ImpressionBloomFilterQueryFeatureHydrator[Query <: PipelineQuery]( + @Named(MemcachedImpressionBloomFilterStore) bloomFilterClient: Store[ + blm.ImpressionBloomFilterKey, + blm.ImpressionBloomFilterSeq + ], + surfaceArea: blm.SurfaceArea) + extends QueryFeatureHydrator[Query] + with Conditionally[Query] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ImpressionBloomFilter") + + override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature) + + override def onlyIf(query: Query): Boolean = query.params(EnableImpressionBloomFilterHydrator) + + override def hydrate(query: Query): Stitch[FeatureMap] = Stitch.callFuture { + bloomFilterClient + .get(blm.ImpressionBloomFilterKey(query.getRequiredUserId, surfaceArea)) + .map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty))) + .map(FeatureMap(ImpressionBloomFilterFeature, _)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ImpressionVideoBloomFilterVideoFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ImpressionVideoBloomFilterVideoFeatureHydrator.scala new file mode 100644 index 000000000..efc4d3ef2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/ImpressionVideoBloomFilterVideoFeatureHydrator.scala @@ -0,0 +1,56 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.storehaus.Store +import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} +import com.twitter.tweet_mixer.model.ModuleNames.MemcachedImpressionVideoBloomFilterStore +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableVideoBloomFilterHydrator +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +object VideoImpressionBloomFilterFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, blm.ImpressionBloomFilterSeq] { + override def defaultValue: blm.ImpressionBloomFilterSeq = + blm.ImpressionBloomFilterSeq(Seq.empty) +} + +@Singleton +case class ImpressionVideoBloomFilterQueryFeatureHydrator @Inject() ( + @Named(MemcachedImpressionVideoBloomFilterStore) bloomFilterClient: Store[ + blm.ImpressionBloomFilterKey, + blm.ImpressionBloomFilterSeq + ]) extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("ImpressionVideoBloomFilter") + + override val features: Set[Feature[_, _]] = Set(VideoImpressionBloomFilterFeature) + + private val SurfaceArea = blm.SurfaceArea.Explore + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableVideoBloomFilterHydrator) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + Stitch.callFuture { + bloomFilterClient + .get(blm.ImpressionBloomFilterKey(userId, SurfaceArea)) + .map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty))) + .map { bloomFilterSeq => + FeatureMapBuilder() + .add(VideoImpressionBloomFilterFeature, bloomFilterSeq).build() + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/LastNonPollingTimeQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/LastNonPollingTimeQueryFeatureHydrator.scala new file mode 100644 index 000000000..eaf347a98 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/LastNonPollingTimeQueryFeatureHydrator.scala @@ -0,0 +1,64 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.user_session_store.ReadRequest +import com.twitter.user_session_store.ReadWriteUserSessionStore +import com.twitter.user_session_store.UserSessionDataset +import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset +import com.twitter.util.Time + +import javax.inject.Inject +import javax.inject.Singleton + +object FollowingLastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] +object LastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] +object NonPollingTimesFeature extends Feature[PipelineQuery, Seq[Long]] + +@Singleton +case class LastNonPollingTimeQueryFeatureHydrator @Inject() ( + userSessionStore: ReadWriteUserSessionStore) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("LastNonPollingTime") + + override val features: Set[Feature[_, _]] = Set( + FollowingLastNonPollingTimeFeature, + LastNonPollingTimeFeature, + NonPollingTimesFeature + ) + + private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.NonPollingTimes) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + userSessionStore + .read(ReadRequest(query.getRequiredUserId, datasets)) + .map { userSession => + val nonPollingTimestamps = userSession.flatMap(_.nonPollingTimestamps) + + val lastNonPollingTime = nonPollingTimestamps + .flatMap(_.nonPollingTimestampsMs.headOption) + .map(Time.fromMilliseconds) + + val followingLastNonPollingTime = nonPollingTimestamps + .flatMap(_.mostRecentHomeLatestNonPollingTimestampMs) + .map(Time.fromMilliseconds) + + val nonPollingTimes = nonPollingTimestamps + .map(_.nonPollingTimestampsMs) + .getOrElse(Seq.empty) + + FeatureMapBuilder() + .add(FollowingLastNonPollingTimeFeature, followingLastNonPollingTime) + .add(LastNonPollingTimeFeature, lastNonPollingTime) + .add(NonPollingTimesFeature, nonPollingTimes) + .build() + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaDeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaDeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.scala new file mode 100644 index 000000000..2d1b694a2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaDeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.scala @@ -0,0 +1,76 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.strato.generated.client.hydra.CachedEgsTweetEmbeddingClientColumn +import com.twitter.strato.columns.hydra.thriftscala.EmbeddingGenerationServiceParamsWithKey +import com.twitter.strato.columns.hydra.thriftscala.EmbeddingGenerationServiceParams +import com.twitter.stitch.Stitch +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaDeepRetrievalModelName + +import javax.inject.Inject +import javax.inject.Singleton + +object MediaDeepRetrievalSignalTweetEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Map[Long, Seq[Int]]] { + override def defaultValue: Map[Long, Seq[Int]] = Map.empty +} + +@Singleton +class MediaDeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory @Inject() ( + cachedEgsTweetEmbeddingClientColumn: CachedEgsTweetEmbeddingClientColumn) { + def build( + signalFn: PipelineQuery => Seq[Long] + ): MediaDeepRetrievalTweetEmbeddingQueryFeatureHydrator = { + new MediaDeepRetrievalTweetEmbeddingQueryFeatureHydrator( + cachedEgsTweetEmbeddingClientColumn, + signalFn) + } +} + +class MediaDeepRetrievalTweetEmbeddingQueryFeatureHydrator( + cachedEgsTweetEmbeddingClientColumn: CachedEgsTweetEmbeddingClientColumn, + signalFn: PipelineQuery => Seq[Long], +) extends QueryFeatureHydrator[PipelineQuery] { + + val modelName: FSParam[String] = MediaDeepRetrievalModelName + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("MediaDeepRetrievalTweetEmbedding") + + override val features: Set[Feature[_, _]] = Set(MediaDeepRetrievalSignalTweetEmbeddingFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val egsParams = EmbeddingGenerationServiceParams( + deepRetrievalModels = Some(true), + modelNames = Some(Seq(query.params(modelName))) + ) + Stitch + .traverse(signalFn(query)) { tweetId => + val egsKey = EmbeddingGenerationServiceParamsWithKey( + tweetId = tweetId, + params = egsParams, + ) + cachedEgsTweetEmbeddingClientColumn.fetcher.fetch(egsKey).flatMap { result => + result.v match { + case Some(embeddingsByModel: Map[String, Seq[Int]]) => + Stitch.value(embeddingsByModel + .get(query.params(modelName)).map(embedding => tweetId -> embedding)) + case other => + Stitch.value(None) + } + } + }.map { results => + val map = results.flatten.toMap + new FeatureMapBuilder() + .add(MediaDeepRetrievalSignalTweetEmbeddingFeature, map) + .build() + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaDeepRetrievalUserEmbeddingQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaDeepRetrievalUserEmbeddingQueryFeatureHydrator.scala new file mode 100644 index 000000000..0deb47628 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaDeepRetrievalUserEmbeddingQueryFeatureHydrator.scala @@ -0,0 +1,121 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.hydra.embedding_generation.{thriftscala => eg} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.stitch.Stitch +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaDeepRetrievalModelName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaEvergreenDeepRetrievalModelName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaPromotedCreatorDeepRetrievalModelName +import com.twitter.tweet_mixer.utils.PipelineFailureCategories.FailedEmbeddingHydrationResponse +import com.twitter.tweet_mixer.utils.PipelineFailureCategories.InvalidEmbeddingHydrationResponse +import javax.inject.Inject +import javax.inject.Singleton + +object MediaDeepRetrievalUserEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Seq[Int]]] { + override def defaultValue: Option[Seq[Int]] = None +} + +@Singleton +class MediaEvergreenUserEmbeddingQueryFeatureHydrator @Inject() ( + egsClient: eg.EmbeddingGenerationService.MethodPerEndpoint) + extends MediaDeepRetrievalUserEmbeddingQueryFeatureHydrator(egsClient) { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("MediaEvergreenDeepRetrievalUserEmbedding") + override val modelName: FSParam[String] = MediaEvergreenDeepRetrievalModelName +} + +@Singleton +class MediaPromotedCreatorUserEmbeddingQueryFeatureHydrator @Inject() ( + egsClient: eg.EmbeddingGenerationService.MethodPerEndpoint) + extends MediaDeepRetrievalUserEmbeddingQueryFeatureHydrator(egsClient) { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("MediaPromotedCreatorDeepRetrievalUserEmbedding") + override val modelName: FSParam[String] = MediaPromotedCreatorDeepRetrievalModelName +} + +@Singleton +class MediaDeepRetrievalUserEmbeddingQueryFeatureHydrator @Inject() ( + egsClient: eg.EmbeddingGenerationService.MethodPerEndpoint) + extends QueryFeatureHydrator[PipelineQuery] { + + val modelName: FSParam[String] = MediaDeepRetrievalModelName + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("MediaDeepRetrievalUserEmbedding") + + override val features: Set[Feature[_, _]] = Set(MediaDeepRetrievalUserEmbeddingFeature) + + private val InvalidMediaResponseException = Stitch.exception( + PipelineFailure( + InvalidEmbeddingHydrationResponse, + "Invalid media embedding hydration response")) + + private val FailedMediaResponseException = Stitch.exception( + PipelineFailure(FailedEmbeddingHydrationResponse, "Failed media embedding hydration response")) + + /** + * The thrift response has been designed to support multiple embeddings for different model configs. For now + * we only expect to use a single config, so we will take the first one and ignore anything else in the response + */ + private object FirstResultInResponse { + def unapply(uer: eg.UserEmbeddingsResponse): Option[eg.UserEmbeddingsResult] = + uer.results.flatMap(_.values.headOption) + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + getEmbedding(query.getRequiredUserId, query.clientContext, query.params(modelName)) + .map { embedding => + new FeatureMapBuilder() + .add(MediaDeepRetrievalUserEmbeddingFeature, embedding) + .build() + } + } + + private def getEmbedding( + userId: Long, + clientContext: ClientContext, + modelName: String + ): Stitch[Option[Seq[Int]]] = { + val egsQuery = eg.EmbeddingsGenerationRequest( + clientContext = ClientContextMarshaller(clientContext), + product = eg.Product.UserEmbeddings, + productContext = Some( + eg.ProductContext.UserEmbeddingsContext( + eg.UserEmbeddingsContext(userIds = Seq(userId), modelNames = Some(Seq(modelName))))) + ) + + Stitch.callFuture(egsClient.generateEmbeddings(egsQuery)).flatMap { + case eg.EmbeddingsGenerationResponse + .UserEmbeddingsResponse(FirstResultInResponse(userEmbeddingsResult)) => + userEmbeddingsResult match { + case eg.UserEmbeddingsResult.UserEmbeddings(embeddings) => + // We expect the embeddings to be keyed on the user id that we passed + embeddings.embeddingsByUserId.flatMap(_.get(userId)) match { + case Some(embeddings) => + Stitch.value(Some(embeddings)) + case _ => Stitch.value(None) + } + case eg.UserEmbeddingsResult.ValidationError(error) => + Stitch.exception( + PipelineFailure( + InvalidEmbeddingHydrationResponse, + error.msg.getOrElse("Unknown validation error in EmbeddingsGenerationService"))) + case _ => InvalidMediaResponseException + } + case _ => FailedMediaResponseException + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaMetadataCandidateFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaMetadataCandidateFeatureHydrator.scala new file mode 100644 index 000000000..d8d033c0f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MediaMetadataCandidateFeatureHydrator.scala @@ -0,0 +1,121 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.stitch.cache.AsyncValueCache +import com.twitter.strato.generated.client.videoRecommendations.twitterClip.TwitterClipClusterIdMhClientColumn +import com.twitter.tweet_mixer.feature.MediaClusterIdFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableDebugMode +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils + +object MediaMetadataCandidateFeatureHydrator { + val memcachePrefix = "MediaMetadata:" + val DefaultFeatureMap = + FeatureMap(MediaClusterIdFeature, None) + private val MemcacheTTLSeconds = Utils.randomizedTTL(10 * 60) +} + +class MediaMetadataCandidateFeatureHydrator( + twitterClipClusterIdMhClientColumn: TwitterClipClusterIdMhClientColumn, + memcache: MemcacheStitchClient, + inMemoryCache: AsyncValueCache[java.lang.Long, Option[Long]], + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + import MediaMetadataCandidateFeatureHydrator._ + + override val features: Set[Feature[_, _]] = + Set(MediaClusterIdFeature) + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("MediaMetadata") + + private val mediaMetadataInMemCacheRequestCounter = + statsReceiver.counter("MediaMetadataInMemCacheRequests") + private val mediaMetadataInMemCacheMissCounter = + statsReceiver.counter("MediaMetadataInMemCacheMisses") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val candidatesBatched = candidates.grouped(1000) + + val featureMapsBatched = candidatesBatched.map { candidatesBatch => + OffloadFuturePools.offloadStitch { + Stitch.traverse(candidatesBatch) { candidate => + mediaMetadataInMemCacheRequestCounter.incr() + val tweetId = candidate.candidate.id + inMemoryCache + .get(tweetId) + .getOrElse(getMediaMetadata(query, tweetId)) + .map(FeatureMap(MediaClusterIdFeature, _)) + .handle { case _ => DefaultFeatureMap } + } + } + } + Stitch.collect(featureMapsBatched.toSeq).map(_.flatten) + } + + private def getMediaMetadata( + query: PipelineQuery, + tweetId: Long + ): Stitch[Option[Long]] = { + mediaMetadataInMemCacheMissCounter.incr() + memcache + .get(memcachePrefix + tweetId.toString) + .flatMap { + case Some(value) => + Stitch + .value(Transformers.deserializeMediaMetadataCacheOption(value)) + .applyEffect { result => + Stitch.async { + inMemoryCache.set(tweetId, Stitch.value(result)) + } + } + case None => + // If debug mode on, wait for strato, otherwise just do it asynchronously. + if (query.params(EnableDebugMode)) readFromStrato(query, tweetId) + else + Stitch.None.applyEffect { _ => + Stitch.async(readFromStrato(query, tweetId)) + } + } + } + + private def readFromStrato( + query: PipelineQuery, + tweetId: Long + ): Stitch[Option[Long]] = { + twitterClipClusterIdMhClientColumn.fetcher + .fetch( + tweetId + ).map(response => response.v).flatMap { + case Some(mediaClusterId) => + val response = Some(mediaClusterId) + val inMemoryCacheSet = inMemoryCache.set(tweetId, Stitch.value(response)) + val memCacheSet = memcache.set( + memcachePrefix + tweetId.toString, + Transformers.serializeMediaMetadataCacheOption(response), + MemcacheTTLSeconds) + Stitch.join(inMemoryCacheSet, memCacheSet) + case _ => + val inMemoryCacheSet = inMemoryCache.set(tweetId, Stitch.value(None)) + val memCacheSet = memcache.set( + memcachePrefix + tweetId.toString, + Transformers.serializeMediaMetadataCacheOption(None), + MemcacheTTLSeconds) + Stitch.join(inMemoryCacheSet, memCacheSet) + }.map { case (response, _) => response } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MultimodalEmbeddingQueryFeatureHydratorFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MultimodalEmbeddingQueryFeatureHydratorFactory.scala new file mode 100644 index 000000000..0148c5f39 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/MultimodalEmbeddingQueryFeatureHydratorFactory.scala @@ -0,0 +1,74 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.content_understanding.PostMultimodalEmbeddingMhClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +object MultimodalEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Map[Long, Option[Seq[Double]]]]] { + override def defaultValue: Option[Map[Long, Option[Seq[Double]]]] = None +} + +@Singleton +class MultimodalEmbeddingQueryFeatureHydratorFactory @Inject() ( + embeddingClientColumn: PostMultimodalEmbeddingMhClientColumn) { + + def build( + signalFn: PipelineQuery => Seq[Long] + ): MultimodalEmbeddingQueryFeatureHydrator = { + new MultimodalEmbeddingQueryFeatureHydrator( + embeddingClientColumn, + signalFn + ) + } +} + +class MultimodalEmbeddingQueryFeatureHydrator( + postMultimodalEmbeddingMhClientColumn: PostMultimodalEmbeddingMhClientColumn, + signalFn: PipelineQuery => Seq[Long]) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("MultimodalEmbedding") + + override val features: Set[Feature[_, _]] = Set(MultimodalEmbeddingFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val tweetIds = signalFn(query) + getEmbedding(tweetIds) + .map { embedding => + new FeatureMapBuilder() + .add(MultimodalEmbeddingFeature, embedding) + .build() + } + } + + private def getEmbedding( + tweetIds: Seq[Long] + ): Stitch[Option[Map[Long, Option[Seq[Double]]]]] = { + // Parallel fetch the embeddings for each tweetId + Stitch + .traverse(tweetIds) { tweetId => + postMultimodalEmbeddingMhClientColumn.fetcher + .fetch(tweetId).map { result => + result.v match { + case Some(tweetEmbedding) if tweetEmbedding.embedding1.isDefined => + tweetEmbedding.embedding1.map(embedding => tweetId -> Some(embedding)) + case other => + Some(tweetId -> None) + } + } + }.map { results => + val map = results.flatten.toMap + if (map.nonEmpty) Some(map) else None + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory.scala new file mode 100644 index 000000000..8010849b1 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory.scala @@ -0,0 +1,159 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.columns.hydra.thriftscala.EmbeddingGenerationServiceParamsWithKey +import com.twitter.strato.columns.hydra.thriftscala.EmbeddingGenerationServiceParams +import com.twitter.strato.generated.client.hydra.CachedEgsTweetEmbeddingClientColumn +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.OutlierDeepRetrievalStratoTimeout +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.OutlierTweetEmbeddingModelNameParam +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.util.JavaTimer +import com.twitter.util.Return +import com.twitter.util.TimeoutException +import com.twitter.util.Timer + +import javax.inject.Inject + +object OutlierDeepRetrievalTweetEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Map[Long, Seq[Int]]]] { + override def defaultValue: Option[Map[Long, Seq[Int]]] = None +} + +class OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory @Inject() ( + stats: StatsReceiver, + cachedEgsTweetEmbeddingClientColumn: CachedEgsTweetEmbeddingClientColumn) { + + def build( + signalFn: PipelineQuery => Seq[Long] + ): OutlierDeepRetrievalEmbeddingQueryFeatureHydrator = { + new OutlierDeepRetrievalEmbeddingQueryFeatureHydrator( + cachedEgsTweetEmbeddingClientColumn, + signalFn, + stats + ) + } +} + +class OutlierDeepRetrievalEmbeddingQueryFeatureHydrator( + cachedEgsTweetEmbeddingClientColumn: CachedEgsTweetEmbeddingClientColumn, + signalFn: PipelineQuery => Seq[Long], + stats: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + + private val statsScope: StatsReceiver = stats.scope("outlier_deep_retrieval_tweet_embedding") + private val requestCounter: Counter = statsScope.counter("requests") + private val successCounter: Counter = statsScope.counter("success") + private val failureCounter: Counter = statsScope.counter("failures") + private val timeoutCounter: Counter = statsScope.counter("timeouts") + private val emptyResultCounter: Counter = statsScope.counter("empty_results") + private val latencyStat: Stat = statsScope.stat("latency_ms") + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "OutlierDeepRetrievalEmbedding") + + override def features: Set[Feature[_, _]] = Set(OutlierDeepRetrievalTweetEmbeddingFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + getEmbedding( + signalFn(query), + query.params(OutlierTweetEmbeddingModelNameParam), + query.params(OutlierDeepRetrievalStratoTimeout)) + .map { embeddingMapOpt => + FeatureMap(OutlierDeepRetrievalTweetEmbeddingFeature, embeddingMapOpt) + } + } + + private def getEmbedding( + tweetIds: Seq[Long], + modelName: String, + outlierDeepRetrievalStratoTimeout: Int + ): Stitch[Option[Map[Long, Seq[Int]]]] = { + val egsParams = EmbeddingGenerationServiceParams( + deepRetrievalModels = Some(true), + modelNames = Some(Seq(modelName)) + ) + val timer: Timer = new JavaTimer(true) + + requestCounter.incr() + + def transformer(batch: Seq[Long]): Future[Seq[(Long, Seq[Int])]] = { + Stitch.run { + Stitch + .collectToTry { + batch.map { tweetId => + val egsKey = EmbeddingGenerationServiceParamsWithKey( + tweetId = tweetId, + params = egsParams + ) + cachedEgsTweetEmbeddingClientColumn.fetcher.fetch(egsKey).flatMap { result => + result.v match { + case Some(embeddingsByModel: Map[String, Seq[Int]]) => + val embedding = embeddingsByModel.get(modelName).map(emb => (tweetId, emb)) + Stitch.value(embedding) + case _ => + Stitch.value(None) + } + } + } + } + .map { results => + results.flatMap { + case Return(Some(embedding)) => Some(embedding) + case _ => None + } + } + } + } + + val batchSize = 32 // Adjust based on performance testing + val startTime = System.currentTimeMillis() + val futureResult = OffloadFuturePools.offloadBatchSeqToFutureSeq( + inputSeq = tweetIds, + transformer = transformer, + batchSize = batchSize, + offload = true // Offload to thread pool + ) + + // Process the result with timeout and stats + Stitch + .callFuture { + futureResult + .within( + timer, + Duration.fromMilliseconds(outlierDeepRetrievalStratoTimeout) + ) + .onSuccess { result => + val latencyMs = (System.currentTimeMillis() - startTime).toFloat + latencyStat.add(latencyMs) // Record latency + if (result.nonEmpty) { + successCounter.incr() + } else { + emptyResultCounter.incr() + } + } + .onFailure { e => + val latencyMs = (System.currentTimeMillis() - startTime).toFloat + latencyStat.add(latencyMs) // Record latency even on failure + failureCounter.incr() + if (e.isInstanceOf[TimeoutException]) { + timeoutCounter.incr() + } + } + } + .map { results => + val idToEmbeddingMap = results.toMap + if (idToEmbeddingMap.nonEmpty) Some(idToEmbeddingMap) else None + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala new file mode 100644 index 000000000..2078b9bd3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala @@ -0,0 +1,60 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.tweet_mixer.feature.RealGraphInNetworkScoresFeature +import com.twitter.tweet_mixer.model.ModuleNames.RealGraphInNetworkScoresOnPremRepo +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.servo.repository.Repository +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.param.EarlybirdInNetworkTweetsParams.EarlybirdInNetworkTweetsEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UTEGEnabled +import com.twitter.wtf.candidate.{thriftscala => wtf} + +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +@Singleton +case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() ( + @Named(RealGraphInNetworkScoresOnPremRepo) realGraphRepo: Repository[Long, Option[ + wtf.CandidateSeq + ]]) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("RealGraphInNetworkScores") + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(UTEGEnabled) || query.params(EarlybirdInNetworkTweetsEnabled) + + override val features: Set[Feature[_, _]] = Set(RealGraphInNetworkScoresFeature) + + private val RealGraphCandidateCount = 1000 + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + + Stitch.callFuture(realGraphRepo.apply(query.getRequiredUserId)).map { realGraphFollowedUsers => + val realGraphScoresFeatures = realGraphFollowedUsers + .map(_.candidates) + .getOrElse(Seq.empty) + .sortBy(-_.score) + .map(candidate => candidate.userId -> scaleScore(candidate.score)) + .take(RealGraphCandidateCount) + .toMap + + FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build() + } + } + + // Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97] + // v1 logic: src/scala/com/twitter/interaction_graph/scalding/jobs/scoring/InteractionGraphScoringJob.scala?L77-80 + private def scaleScore(score: Double): Double = + if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score + +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/RequestCountryPlaceIdFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/RequestCountryPlaceIdFeatureHydrator.scala new file mode 100644 index 000000000..8ada20753 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/RequestCountryPlaceIdFeatureHydrator.scala @@ -0,0 +1,54 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.geoduck.util.country.CountryInfo +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.RequestCountryPlaceIdFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableRequestCountryPlaceIdHydrator +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class RequestPlaceIdsQueryFeatureHydrator @Inject() (statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { + private def UnknownCountryPlaceId: Long = -1L + + private val scopedStats = statsReceiver.scope(getClass.getSimpleName) + private val emptyPlaceIdStats = scopedStats.counter("emptyPlaceIdStats") + private val noCountryCodeStats = scopedStats.counter("noCountryCodeStats") + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RequestPlaceIds") + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableRequestCountryPlaceIdHydrator) + + override def features: Set[Feature[_, _]] = Set(RequestCountryPlaceIdFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + query.getCountryCode match { + case Some(countryCode) => + val placeId: Long = getCountryPlaceId(countryCode).getOrElse(UnknownCountryPlaceId) + if (placeId == UnknownCountryPlaceId) emptyPlaceIdStats.incr() + Stitch.value(FeatureMapBuilder().add(RequestCountryPlaceIdFeature, placeId).build()) + + case _ => + noCountryCodeStats.incr() + Stitch.value( + FeatureMapBuilder().add(RequestCountryPlaceIdFeature, UnknownCountryPlaceId).build()) + } + } + + private def getCountryPlaceId(countryCode: String): Option[Long] = { + CountryInfo.lookupByCode(countryCode).map { countryInfo => + countryInfo.placeIdLong + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/SGSFollowedUsersQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/SGSFollowedUsersQueryFeatureHydrator.scala new file mode 100644 index 000000000..ab39efa5d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/SGSFollowedUsersQueryFeatureHydrator.scala @@ -0,0 +1,46 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.socialgraph.{thriftscala => sg} +import com.twitter.stitch.Stitch +import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient} +import javax.inject.Inject +import javax.inject.Singleton + +object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]] + +@Singleton +case class SGSFollowedUsersQueryFeatureHydrator @Inject() ( + socialGraphStitchClient: SocialGraphStitchClient) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("SGSFollowedUsers") + + override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature) + + private val SocialGraphLimit = 2999 + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + + val request = sg.IdsRequest( + relationships = Seq( + sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true), + sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false) + ), + pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit))) + ) + + socialGraphStitchClient + .ids(request).map(_.ids) + .map { followedUsers => + FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build() + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/SignalInfoCandidateFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/SignalInfoCandidateFeatureHydrator.scala new file mode 100644 index 000000000..e304b39bd --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/SignalInfoCandidateFeatureHydrator.scala @@ -0,0 +1,61 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.SearcherRealtimeHistorySourceSignalFeature +import com.twitter.tweet_mixer.feature.SignalInfo +import com.twitter.tweet_mixer.feature.SourceSignalFeature +import com.twitter.tweet_mixer.feature.USSFeatures +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableSignalInfoHydrator + +object SignalInfoFeature extends Feature[TweetCandidate, Seq[SignalInfo]] + +object SignalInfoCandidateFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("SignalType") + + override val features: Set[Feature[_, _]] = Set(SignalInfoFeature) + + override def onlyIf(input: PipelineQuery): Boolean = { + input.params(EnableSignalInfoHydrator) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val signalInfoFeatureMap = USSFeatures + .getSignalsWithInfo[Long]( + query, + USSFeatures.TweetFeatures ++ USSFeatures.ProducerFeatures ++ USSFeatures.ImmersiveVideoTweetFeatures + ) + + val searcherRealtimeHistoryFeatureMap = + USSFeatures.getSignalsWithInfo[String](query, USSFeatures.SearcherRealtimeHistoryFeatures) + + val userId = query.getRequiredUserId + + Stitch.value(candidates.map { candidate => + val sourceSignal = candidate.features.getOrElse(SourceSignalFeature, userId) + val signalInfo = signalInfoFeatureMap.getOrElse(sourceSignal, Seq.empty) + val searcherRealtimeHistorySourceSignal = + candidate.features.getOrElse(SearcherRealtimeHistorySourceSignalFeature, "") + val searcherRealtimeHistorySignalInfo = + searcherRealtimeHistoryFeatureMap.getOrElse(searcherRealtimeHistorySourceSignal, Seq.empty) + createFeatureMap(signalInfo ++ searcherRealtimeHistorySignalInfo) + }) + } + + private def createFeatureMap(signalInfo: Seq[SignalInfo]): FeatureMap = + FeatureMap(SignalInfoFeature, signalInfo) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TweetypieCandidateFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TweetypieCandidateFeatureHydrator.scala new file mode 100644 index 000000000..a96cc2a85 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TweetypieCandidateFeatureHydrator.scala @@ -0,0 +1,311 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.mediaservices.commons.tweetmedia.thriftscala.MediaInfo +import com.twitter.mediaservices.commons.tweetmedia.thriftscala.MediaSizeType +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.spam.rtf.thriftscala.SafetyLevel +import com.twitter.stitch.Stitch +import com.twitter.stitch.cache.AsyncValueCache +import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} +import com.twitter.strato.client.Fetcher +import com.twitter.tweet_mixer.feature.TweetInfoFeatures._ +import com.twitter.tweet_mixer.feature.AuthorIdFeature +import com.twitter.tweet_mixer.feature.FromInNetworkSourceFeature +import com.twitter.tweet_mixer.feature.TweetBooleanInfoFeature +import com.twitter.tweet_mixer.feature.TweetInfoFeatures +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MinVideoDurationParam +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import com.twitter.tweet_mixer.utils.Transformers +import com.twitter.tweet_mixer.utils.Utils +import com.twitter.tweetypie.thriftscala.TweetVisibilityPolicy +import com.twitter.tweetypie.{thriftscala => TP} +import com.twitter.mediaservices.commons.{thriftscala => mc} +import com.twitter.strato.client.Client +import com.twitter.strato.generated.client.tweetypie.managed.TweetMixerTesOnTweetClientColumn +import com.twitter.tweet_mixer.feature.MediaIdFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableTweetEntityServiceMigrationDiffy +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LongFormMaxAspectRatioParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LongFormMaxDurationParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LongFormMinAspectRatioParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LongFormMinDurationParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LongFormMinHeightParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LongFormMinWidthParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ShortFormMaxAspectRatioParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ShortFormMaxDurationParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ShortFormMinAspectRatioParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ShortFormMinDurationParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ShortFormMinHeightParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ShortFormMinWidthParam + +object TweetypieCandidateFeatureHydrator { + import scala.language.implicitConversions + + val CoreTweetFields: Set[TP.TweetInclude] = Set[TP.TweetInclude]( + TP.TweetInclude.TweetFieldId(TP.Tweet.CoreDataField.id), + TP.TweetInclude.TweetFieldId(TP.Tweet.MediaField.id), + TP.TweetInclude.TweetFieldId(TP.Tweet.MediaKeysField.id), + TP.TweetInclude.TweetFieldId(TP.Tweet.UrlsField.id) + ) + + val DefaultFeatureMap = + FeatureMap(TweetBooleanInfoFeature, None, AuthorIdFeature, None, MediaIdFeature, None) + + implicit def bool2int(b: Boolean) = if (b) 1 else 0 + + private val TTLSeconds = Utils.randomizedTTL(10 * 60) +} + +class TweetypieCandidateFeatureHydrator( + tweetypieStitchClient: TweetypieStitchClient, + safetyLevelPredicate: PipelineQuery => SafetyLevel, + memcache: MemcacheStitchClient, + inMemoryCache: AsyncValueCache[java.lang.Long, Option[(Int, Long, Long)]], + statsReceiver: StatsReceiver, + stratoClient: Client) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + import TweetypieCandidateFeatureHydrator._ + + override val features: Set[Feature[_, _]] = Set( + TweetBooleanInfoFeature, + AuthorIdFeature, + MediaIdFeature + ) + + private lazy val tesFetcher = new TweetMixerTesOnTweetClientColumn(stratoClient).fetcher + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie") + + private val inMemoryCacheRequestCounter = statsReceiver.counter("InMemCacheRequests") + private val inMemoryCacheMissCounter = statsReceiver.counter("InMemCacheMisses") + + private def getCacheKey(tweetId: Long): String = { + "Tweetypie:" + tweetId.toString + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val candidatesBatched = candidates.grouped(1000) + + val featureMapsBatched = candidatesBatched.map { candidatesBatch => + OffloadFuturePools.offloadStitch { + Stitch.traverse(candidatesBatch) { candidate => + val fromInNetwork = candidate.features.getOrElse(FromInNetworkSourceFeature, false) + if (fromInNetwork) { + val featureMap = { + FeatureMap( + TweetBooleanInfoFeature, + None, + AuthorIdFeature, + candidate.features.getOrElse(AuthorIdFeature, None), + MediaIdFeature, + None + ) + } + Stitch.value(featureMap) + } else { + inMemoryCacheRequestCounter.incr() + val tweetId = candidate.candidate.id + inMemoryCache + .get(tweetId) + .getOrElse(getTweetInfo(query, tweetId)) + .map(createFeatureMap(_)) + .handle { case _ => DefaultFeatureMap } + } + } + } + } + Stitch.collect(featureMapsBatched.toSeq).map(_.flatten) + } + + private def getTweetInfo( + query: PipelineQuery, + tweetId: Long + ): Stitch[Option[(Int, Long, Long)]] = { + inMemoryCacheMissCounter.incr() + memcache + .get(getCacheKey(tweetId)) + .flatMap { + case Some(value) => + Stitch + .value(Transformers.deserializeTpCacheOption(value)) + .applyEffect { result => + Stitch.async { + inMemoryCache.set(tweetId, Stitch.value(result)) + } + } + case None => + if (query.params(EnableTweetEntityServiceMigrationDiffy)) { + Stitch.None.applyEffect { _ => + Stitch.async(readFromTweetEntityService(query, tweetId, tesFetcher)) + } + } else { + Stitch.None.applyEffect { _ => + Stitch.async(readFromTweetypie(query, tweetId)) + } + } + } + } + + private def createFeatureMap(tweetypieResponse: Option[(Int, Long, Long)]): FeatureMap = { + val tweetBooleanInfo = tweetypieResponse.map(_._1) + val authorId = tweetypieResponse.map(_._2) + val mediaId = tweetypieResponse.map(_._3) + FeatureMap( + TweetBooleanInfoFeature, + tweetBooleanInfo, + AuthorIdFeature, + authorId, + MediaIdFeature, + mediaId + ) + } + + private def processTweet(query: PipelineQuery, tweet: TP.Tweet): Option[(Int, Long, Long)] = { + val coreData = tweet.coreData + val isReply = coreData.exists(_.reply.nonEmpty) + val isRetweet = coreData.exists(_.share.isDefined) + + val hasMultipleMedia = + tweet.mediaKeys.exists(_.map(_.mediaCategory).size > 1) + + val durationInfo = tweet.media.flatMap(_.flatMap { + _.mediaInfo match { + case Some(MediaInfo.VideoInfo(info)) => + Some((info.durationMillis + 999) / 1000) // video playtime always round up + case _ => None + } + }.headOption) + + val hasImage = + tweet.mediaKeys.exists(_.exists(_.mediaCategory == mc.MediaCategory.TweetImage)) + + val hasVideo = durationInfo.isDefined + + val isLongVideo = durationInfo.exists(_ > query.params(MinVideoDurationParam).inSeconds) + + val mediaDimensionsOpt = + tweet.media.flatMap( + _.headOption.flatMap(_.sizes + .find(_.sizeType == MediaSizeType.Orig).map(size => (size.width, size.height)))) + + val mediaWidth = mediaDimensionsOpt.map(_._1).getOrElse(1) + val mediaHeight = mediaDimensionsOpt.map(_._2).getOrElse(1) + val aspectRatio: Double = mediaWidth.toDouble / math.max(1.0, mediaHeight.toDouble) + // high resolution media's width is always greater than 480px and height is always greater than 480px + val isHighMediaResolution = mediaHeight > 480 && mediaWidth > 480 + + val isLandscapeVideo = mediaWidth >= mediaHeight + + val hasUrl = tweet.urls.exists(_.nonEmpty) + + val isLongFormVideo = + hasVideo && aspectRatio >= query.params(LongFormMinAspectRatioParam) && aspectRatio <= query + .params(LongFormMaxAspectRatioParam) && durationInfo.exists( + _ >= query.params(LongFormMinDurationParam).inSeconds) && durationInfo.exists( + _ <= query.params(LongFormMaxDurationParam).inSeconds) && mediaWidth >= query.params( + LongFormMinWidthParam) && mediaHeight >= query.params(LongFormMinHeightParam) + + val isShortFormVideo = + hasVideo && aspectRatio >= query.params(ShortFormMinAspectRatioParam) && aspectRatio <= query + .params(ShortFormMaxAspectRatioParam) && durationInfo.exists( + _ >= query.params(ShortFormMinDurationParam).inSeconds) && durationInfo.exists( + _ <= query.params(ShortFormMaxDurationParam).inSeconds) && mediaWidth >= query.params( + ShortFormMinWidthParam) && mediaHeight >= query.params(ShortFormMinHeightParam) + + val tweetBooleanInfo = List( + IsReply -> isReply, + HasMultipleMedia -> hasMultipleMedia, + HasVideo -> hasVideo, + IsHighMediaResolution -> isHighMediaResolution, + HasUrl -> hasUrl, + IsLongVideo -> isLongVideo, + IsLandscapeVideo -> isLandscapeVideo, + IsRetweet -> isRetweet, + HasImage -> hasImage, + IsShortFormVideo -> isShortFormVideo, + IsLongFormVideo -> isLongFormVideo + ).foldLeft(0)((acc, feature) => + if (feature._2) TweetInfoFeatures.setFeature(feature._1, acc) else acc) + + val authorIdOpt = coreData.map(_.userId) + val mediaId = tweet.media.flatMap(_.headOption).map(_.mediaId).getOrElse(0L) + + authorIdOpt.map((tweetBooleanInfo, _, mediaId)) + } + + private def readFromTweetEntityService( + query: PipelineQuery, + tweetId: Long, + fetcher: Fetcher[Long, Unit, TP.Tweet] + ): Stitch[Option[(Int, Long, Long)]] = { + + val fetchResult: Stitch[Option[TP.Tweet]] = fetcher.fetch(tweetId, ()).map(_.v) + + fetchResult + .flatMap { + case Some(tweet) => + val tweetData = processTweet(query, tweet) + val inMemoryCacheSet = inMemoryCache.set(tweetId, Stitch.value(tweetData)) + val memCacheSet = memcache.set( + getCacheKey(tweetId), + Transformers.serializeTpCacheOption(tweetData), + TTLSeconds + ) + Stitch.join(inMemoryCacheSet, memCacheSet) + case _ => + val inMemoryCacheSet = inMemoryCache.set(tweetId, Stitch.value(None)) + val memCacheSet = memcache.set( + getCacheKey(tweetId), + Transformers.serializeTpCacheOption(None), + TTLSeconds + ) + Stitch.join(inMemoryCacheSet, memCacheSet) + }.map { case (tweetResponse, _) => tweetResponse } + } + + private def readFromTweetypie( + query: PipelineQuery, + tweetId: Long + ): Stitch[Option[(Int, Long, Long)]] = { + tweetypieStitchClient + .getTweetFields( + tweetId = tweetId, + options = TP.GetTweetFieldsOptions( + tweetIncludes = CoreTweetFields, + includeRetweetedTweet = false, + includeQuotedTweet = false, + visibilityPolicy = TweetVisibilityPolicy.UserVisible, + safetyLevel = Some(safetyLevelPredicate(query)) + ) + ) + .flatMap { + case TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), _, _) => + val tweet = found.tweet + val tweetypieResponse = processTweet(query, tweet) + val inMemoryCacheSet = inMemoryCache.set(tweetId, Stitch.value(tweetypieResponse)) + val memCacheSet = memcache.set( + getCacheKey(tweetId), + Transformers.serializeTpCacheOption(tweetypieResponse), + TTLSeconds) + Stitch.join(inMemoryCacheSet, memCacheSet) + case _ => + val inMemoryCacheSet = inMemoryCache.set(tweetId, Stitch.value(None)) + val memCacheSet = memcache.set( + getCacheKey(tweetId), + Transformers.serializeTpCacheOption(None), + TTLSeconds) + Stitch.join(inMemoryCacheSet, memCacheSet) + }.map { case (tweetypieResponse, _) => tweetypieResponse } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TweetypieSeedTweetsQueryFeatureHydratorFactory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TweetypieSeedTweetsQueryFeatureHydratorFactory.scala new file mode 100644 index 000000000..67f77bcbc --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TweetypieSeedTweetsQueryFeatureHydratorFactory.scala @@ -0,0 +1,151 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} +import com.twitter.tweet_mixer.feature.AuthorIdFeature +import com.twitter.tweetypie.thriftscala.TweetVisibilityPolicy +import com.twitter.tweetypie.{thriftscala => TP} +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton +import com.twitter.strato.client.Client +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.tweetypie.managed.TweetMixerSeedOnTweetClientColumn +import com.twitter.strato.generated.client.tweetypie.managed.TweetMixerSeedTesOnTweetClientColumn +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableTweetEntityServiceMigration +import com.twitter.finagle.stats.StatsReceiver + +object SeedsTextFeatures extends Feature[PipelineQuery, Option[Map[Long, String]]] + +@Singleton +class TweetypieSeedTweetsQueryFeatureHydratorFactory @Inject() ( + tweetypieStitchClient: TweetypieStitchClient, + statsReceiver: StatsReceiver, + @Named("StratoClientWithModerateTimeout") stratoClient: Client) { + + private val tesFetcher = new TweetMixerSeedTesOnTweetClientColumn(stratoClient).fetcher + private val tesDiffyFetcher = new TweetMixerSeedOnTweetClientColumn(stratoClient).fetcher + + def build(signalFn: PipelineQuery => Seq[Long]): TweetypieSeedTweetsQueryFeatureHydrator = { + new TweetypieSeedTweetsQueryFeatureHydrator( + tweetypieStitchClient, + signalFn, + statsReceiver, + tesFetcher, + tesDiffyFetcher) + } +} + +class TweetypieSeedTweetsQueryFeatureHydrator( + tweetypieStitchClient: TweetypieStitchClient, + signalFn: PipelineQuery => Seq[Long], + statsReceiver: StatsReceiver, + tesFetcher: Fetcher[Long, TP.GetTweetFieldsOptions, TP.GetTweetFieldsResult], + tesDiffyFetcher: Fetcher[Long, TP.GetTweetFieldsOptions, TP.GetTweetFieldsResult]) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "TweetypieSeedTweets") + + override def features: Set[Feature[_, _]] = Set(SeedsTextFeatures, AuthorIdFeature) + + private val TweetLinkUrl = "".r + + private val tesSeedTweetsFoundCounter = statsReceiver.counter("TesSeedTweetsFound") + private val tesSeedTweetsNotFoundCounter = statsReceiver.counter("TesSeedTweetsNotFound") + private val tweetypieSeedTweetsFoundCounter = statsReceiver.counter("TweetypieSeedTweetsFound") + private val tweetypieSeedTweetsNotFoundCounter = + statsReceiver.counter("TweetypieSeedTweetsNotFound") + + private val getTweetFieldsOption = TP.GetTweetFieldsOptions( + tweetIncludes = Set(TP.TweetInclude.TweetFieldId(TP.Tweet.CoreDataField.id)), + includeRetweetedTweet = false, + includeQuotedTweet = false, + visibilityPolicy = TweetVisibilityPolicy.UserVisible, + safetyLevel = None + ) + + private def readFromTweetypie(tweetIds: Seq[Long]): Stitch[FeatureMap] = { + Stitch + .traverse(tweetIds) { tweetId => + tweetypieStitchClient + .getTweetFields( + tweetId = tweetId, + options = getTweetFieldsOption, + ) + .map { + case TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), _, _) => + tweetypieSeedTweetsFoundCounter.incr() + val textOpt = found.tweet.coreData.map(_.text) + val authorIdOpt = found.tweet.coreData.map(_.userId) + (tweetId, textOpt, authorIdOpt) + case _ => + tweetypieSeedTweetsNotFoundCounter.incr() + (tweetId, None, None) + } + }.map { idTextAndAuthor: Seq[(Long, Option[String], Option[Long])] => + val idToText = idTextAndAuthor.collect { + case (id, Some(text), _) => (id, TweetLinkUrl.replaceAllIn(text, "").trim) + }.toMap + + val authorIds = idTextAndAuthor.collect { + case (_, _, Some(authorId)) => authorId + } + + FeatureMap( + SeedsTextFeatures, + Some(idToText), + AuthorIdFeature, + authorIds.headOption + ) + } + } + + private def readFromTes(tweetIds: Seq[Long]): Stitch[FeatureMap] = { + Stitch + .traverse(tweetIds) { tweetId => + tesDiffyFetcher + .fetch(tweetId, getTweetFieldsOption).map(_.v).map { + case Some(TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), _, _)) => + tesSeedTweetsFoundCounter.incr() + val textOpt = found.tweet.coreData.map(_.text) + val authorIdOpt = found.tweet.coreData.map(_.userId) + (tweetId, textOpt, authorIdOpt) + case _ => + tesSeedTweetsNotFoundCounter.incr() + (tweetId, None, None) + } + }.map { idTextAndAuthor: Seq[(Long, Option[String], Option[Long])] => + val idToText = idTextAndAuthor.collect { + case (id, Some(text), _) => (id, TweetLinkUrl.replaceAllIn(text, "").trim) + }.toMap + + val authorIds = idTextAndAuthor.collect { + case (_, _, Some(authorId)) => authorId + } + + FeatureMap( + SeedsTextFeatures, + Some(idToText), + AuthorIdFeature, + authorIds.headOption + ) + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val tweetIds = signalFn(query) + + if (query.params(EnableTweetEntityServiceMigration)) { + readFromTes(tweetIds) + } else { + readFromTweetypie(tweetIds) + } + + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TwhinUserPositiveEmbeddingQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TwhinUserPositiveEmbeddingQueryFeatureHydrator.scala new file mode 100644 index 000000000..fc2c336ce --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/TwhinUserPositiveEmbeddingQueryFeatureHydrator.scala @@ -0,0 +1,51 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.tweet_mixer.model.ModuleNames.TwhinRebuildUserPositiveEmbeddingsStore +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +object TwhinUserPositiveEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Seq[Double]]] { + override def defaultValue: Option[Seq[Double]] = None +} + +@Singleton +class TwhinUserPositiveEmbeddingQueryFeatureHydrator @Inject() ( + @Named(TwhinRebuildUserPositiveEmbeddingsStore) store: ReadableStore[ + (Long, Long), + TwhinTweetEmbedding + ]) extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TwhinUserPositiveEmbedding") + + override val features: Set[Feature[_, _]] = Set(TwhinUserPositiveEmbeddingFeature) + + private def l2Normalize(v: Seq[Double]): Seq[Double] = { + val sumSquares = v.foldLeft(0.0)((acc, x) => acc + x * x) + if (sumSquares <= 0.0) v + else { + val n = math.sqrt(sumSquares) + v.map(_ / n) + } + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + Stitch.callFuture(store.get((query.getRequiredUserId, 1L))).map { resultOpt => + val twhinUserPositiveEmbedding: Option[Seq[Double]] = resultOpt.map(_.embedding) + FeatureMapBuilder() + .add(TwhinUserPositiveEmbeddingFeature, twhinUserPositiveEmbedding.map(l2Normalize)).build() + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSDeepRetrievalTweetEmbeddingFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSDeepRetrievalTweetEmbeddingFeatureHydrator.scala new file mode 100644 index 000000000..fcf37ac7d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSDeepRetrievalTweetEmbeddingFeatureHydrator.scala @@ -0,0 +1,266 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.util.Duration +import com.twitter.util.Return +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.USSDeepRetrievalI2iEmbModelName +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.USSDeepRetrievalSimilarityTweetTweetANNScoreThreshold +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.USSDeepRetrievalStratoTimeout +import com.twitter.strato.generated.client.hydra.CachedEgsTweetEmbeddingClientColumn +import com.twitter.strato.columns.hydra.thriftscala.EmbeddingGenerationServiceParamsWithKey +import com.twitter.strato.columns.hydra.thriftscala.EmbeddingGenerationServiceParams +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.Stat +import javax.inject.Inject +import javax.inject.Singleton +import com.twitter.tweet_mixer.feature.SignalInfo +import breeze.linalg.DenseVector +import breeze.linalg.norm +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableUSSDeepRetrievalTweetEmbeddingFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.tweet_mixer.feature.USSFeatures._ +import com.twitter.util.Future +import com.twitter.util.JavaTimer +import com.twitter.util.TimeoutException +import com.twitter.util.Timer + +object USSDeepRetrievalTweetEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Map[Long, Seq[Int]]]] { + override def defaultValue: Option[Map[Long, Seq[Int]]] = None +} + +@Singleton +class USSDeepRetrievalTweetEmbeddingFeatureHydrator @Inject() ( + cachedEgsTweetEmbeddingClientColumn: CachedEgsTweetEmbeddingClientColumn, + stats: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { + + private val statsScope: StatsReceiver = stats.scope("uss_deep_retrieval_tweet_embedding") + private val overallFilterRateStat: Stat = statsScope.stat("overall_filter_rate") + private val similarityGreaterThanEqualThresholdCounter: Counter = + statsScope.counter("similarity_greater_than_equal_thresholdCounter") + private val similarityLessThanThresholdCounter: Counter = + statsScope.counter("similarity_less_than_thresholdCounter") + private val requestCounter: Counter = statsScope.counter("requests") + private val successCounter: Counter = statsScope.counter("success") + private val failureCounter: Counter = statsScope.counter("failures") + private val timeoutCounter: Counter = statsScope.counter("timeouts") + private val emptyResultCounter: Counter = statsScope.counter("empty_results") + private val latencyStat: Stat = statsScope.stat("latency_ms") + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("USSDeepRetrievalTweetEmbedding") + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableUSSDeepRetrievalTweetEmbeddingFeatureHydrator) + + override val features: Set[Feature[_, _]] = TweetFeatures.asInstanceOf[Set[Feature[_, _]]] + + private lazy val emptyFeatureMap: FeatureMap = { + val emptyFeatureMapBuilder = FeatureMapBuilder() + TweetFeatures.foreach { feature => + emptyFeatureMapBuilder.add(feature, Map.empty[Long, Seq[SignalInfo]]) + } + emptyFeatureMapBuilder.build() + } + + def computeCosineSimilarity(embedding1: Seq[Int], embedding2: Seq[Int]): Double = { + val v1 = + if (embedding1.nonEmpty) + DenseVector(embedding1.map(_.toDouble).toArray) + else DenseVector.zeros[Double](256) + val v2 = + if (embedding2.nonEmpty) + DenseVector(embedding2.map(_.toDouble).toArray) + else DenseVector.zeros[Double](256) + + val dotProduct = v1 dot v2 + val denom = norm(v1) * norm(v2) + if (denom == 0.0) 0.0 else dotProduct / denom + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + // Extract tweet IDs from positive tweet features + val tweetIdsFromPositiveSignals = TweetFeatures.flatMap { feature => + feature.getValue(query).keys + }.toSeq + + val tweetIdsFromNegativeSignals = NegativeFeaturesTweetBased.flatMap { feature => + feature.getValue(query).keys + }.toSeq + + val combinedTweetIds = tweetIdsFromPositiveSignals ++ tweetIdsFromNegativeSignals + + if (combinedTweetIds.isEmpty) { + Stitch.value(emptyFeatureMap) + } else { + getEmbedding( + combinedTweetIds, + query.clientContext, + query.params(USSDeepRetrievalI2iEmbModelName), + query.params(USSDeepRetrievalStratoTimeout) + ).map { embedding => + val similarityThreshold = + query.params(USSDeepRetrievalSimilarityTweetTweetANNScoreThreshold) + + // Pair negative embeddings with their tweet IDs + val negativeEmbeddingsWithIds = embedding match { + case Some(map) => + map.collect { + case (tweetId, emb) if tweetIdsFromNegativeSignals.contains(tweetId) => + (tweetId, emb) + }.toSeq + case None => Seq.empty + } + + var totalSignalsBeforeCount = 0 + var totalSignalsAfterCount = 0 + + val filteredTweetFeatures = TweetFeatures.map { feature => + val signals = feature.getValue(query) + val signalsBeforeCount = signals.size + totalSignalsBeforeCount += signalsBeforeCount + + val filteredSignals = signals.filter { + case (tweetId, _) => + embedding.getOrElse(Map.empty).get(tweetId) match { + case Some(emb) => + val isDissimilar = if (negativeEmbeddingsWithIds.isEmpty) { + true + } else { + negativeEmbeddingsWithIds.exists { + case (_, negEmb) => + val similarity = computeCosineSimilarity(emb, negEmb) + if (similarity > similarityThreshold) + similarityGreaterThanEqualThresholdCounter.incr() + else + similarityLessThanThresholdCounter.incr() + similarity < similarityThreshold + } + } + isDissimilar + case None => + true + } + } + val signalsAfterCount = filteredSignals.size + totalSignalsAfterCount += signalsAfterCount + + feature -> filteredSignals + }.toMap + + // Record overall filter rate + if (totalSignalsBeforeCount > 0) { + val overallFilterRate = + ((totalSignalsBeforeCount - totalSignalsAfterCount).toDouble / totalSignalsBeforeCount) * 100.0 + overallFilterRateStat.add(overallFilterRate.toFloat) + } else { + overallFilterRateStat.add(-1.0f) + } + + val featureMapBuilder = FeatureMapBuilder() + filteredTweetFeatures.foreach { + case (feature, signals) => + featureMapBuilder.add(feature, signals) + } + featureMapBuilder.build() + } + } + } + + private def getEmbedding( + tweetIds: Seq[Long], + clientContext: ClientContext, + modelName: String, + ussDeepRetrievalStratoTimeout: Int + ): Stitch[Option[Map[Long, Seq[Int]]]] = { + val egsParams = EmbeddingGenerationServiceParams( + deepRetrievalModels = Some(true), + modelNames = Some(Seq(modelName)) + ) + val timer: Timer = new JavaTimer(true) + + requestCounter.incr() + + def transformer(batch: Seq[Long]): Future[Seq[(Long, Seq[Int])]] = { + Stitch.run { + Stitch + .collectToTry { + batch.map { tweetId => + val egsKey = EmbeddingGenerationServiceParamsWithKey( + tweetId = tweetId, + params = egsParams + ) + cachedEgsTweetEmbeddingClientColumn.fetcher.fetch(egsKey).flatMap { result => + result.v match { + case Some(embeddingsByModel: Map[String, Seq[Int]]) => + val embedding = embeddingsByModel.get(modelName).map(emb => (tweetId, emb)) + Stitch.value(embedding) + case _ => + Stitch.value(None) + } + } + } + } + .map { results => + results.flatMap { + case Return(Some(embedding)) => Some(embedding) + case _ => None + } + } + } + } + + val batchSize = 32 // Adjust based on performance testing + val startTime = System.currentTimeMillis() + val futureResult = OffloadFuturePools.offloadBatchSeqToFutureSeq( + inputSeq = tweetIds, + transformer = transformer, + batchSize = batchSize, + offload = true // Offload to thread pool + ) + + // Process the result with timeout and stats + Stitch + .callFuture { + futureResult + .within( + timer, + Duration.fromMilliseconds(ussDeepRetrievalStratoTimeout), + new TimeoutException("Embedding fetch timed out after 100ms") + ) + .onSuccess { result => + val latencyMs = (System.currentTimeMillis() - startTime).toFloat + latencyStat.add(latencyMs) // Record latency + if (result.nonEmpty) { + successCounter.incr() + } else { + emptyResultCounter.incr() + } + } + .onFailure { e => + val latencyMs = (System.currentTimeMillis() - startTime).toFloat + latencyStat.add(latencyMs) // Record latency even on failure + failureCounter.incr() + if (e.isInstanceOf[TimeoutException]) { + timeoutCounter.incr() + } + } + } + .map { results => + val idToEmbeddingMap = results.toMap + if (idToEmbeddingMap.nonEmpty) Some(idToEmbeddingMap) else None + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSGrokCategoryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSGrokCategoryFeatureHydrator.scala new file mode 100644 index 000000000..0d582c124 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSGrokCategoryFeatureHydrator.scala @@ -0,0 +1,214 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.content_understanding.api.PostAnnotationsOnTweetClientColumn +import com.twitter.tweet_mixer.feature._ +import com.twitter.tweet_mixer.feature.USSFeatures._ +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableUSSGrokCategoryFeatureHydrator +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.mutable + +object USSGrokCategoryFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Map[Long, Int]]] { + override def defaultValue: Option[Map[Long, Int]] = None +} + +@Singleton +class USSGrokCategoryFeatureHydratorFactory @Inject() ( + postAnnotationsOnTweetClientColumn: PostAnnotationsOnTweetClientColumn) { + + def build( + signalFn: PipelineQuery => Seq[Long] + ): USSGrokCategoryFeatureHydrator = { + new USSGrokCategoryFeatureHydrator( + postAnnotationsOnTweetClientColumn, + signalFn + ) + } +} + +@Singleton +class USSGrokCategoryFeatureHydrator @Inject() ( + postAnnotationsOnTweetClientColumn: PostAnnotationsOnTweetClientColumn, + signalFn: PipelineQuery => Seq[Long]) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("USSGrokCategory") + + override val features: Set[Feature[_, _]] = + Set(USSGrokCategoryFeature) ++ TweetFeatures + + private val fetcher = postAnnotationsOnTweetClientColumn.fetcher + + private val MaxTweetsPerCategory = 15 + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableUSSGrokCategoryFeatureHydrator) + + private def diversifySignalsByCategory( + query: PipelineQuery, + categoryMap: Map[Long, Int], + maxTweetsPerCategory: Int = MaxTweetsPerCategory + ): Map[USSFeature[Long], Map[Long, Seq[SignalInfo]]] = { + + val allSignals = TweetFeatures.map { feature => + val signals = feature.getValue(query) + feature -> signals + }.toMap + + // 1. construct tweet to uss map + val tweetToFeaturesMap = mutable.Map[Long, mutable.Set[USSFeature[Long]]]() + allSignals.foreach { + case (feature, signalMap) => + signalMap.keys.foreach { tweetId => + if (!tweetToFeaturesMap.contains(tweetId)) { + tweetToFeaturesMap(tweetId) = mutable.Set[USSFeature[Long]]() + } + tweetToFeaturesMap(tweetId) += feature + } + } + + // 2. construct category to tweets map + val categoryToTweetsMap = categoryMap.groupBy(_._2).map { + case (category, tweets) => + category -> tweets.keys.toSeq.take(maxTweetsPerCategory) + } + + // 3. construct category queues based on the categoryToTweetsMap + val categoryQueues = mutable.ArrayBuffer[mutable.Queue[Long]]() + val categoryToQueueIndex = mutable.Map.empty[Int, Int] + + // Create a queue for each category and track its index + categoryMap.foreach { + case (tweetId, category) => + // only process tweets that exist in at least one feature + if (tweetToFeaturesMap.contains(tweetId)) { + if (!categoryToQueueIndex.contains(category)) { + categoryQueues.append(mutable.Queue[Long]()) + categoryToQueueIndex.put(category, categoryQueues.length - 1) + } + + // only add tweets that are in the maxTweetsPerCategory limit + val tweetsInCategory = categoryToTweetsMap.getOrElse(category, Seq.empty) + if (tweetsInCategory.contains(tweetId)) { + categoryQueues(categoryToQueueIndex(category)).enqueue(tweetId) + } + } + } + + // 4. count the number of tweets assigned to each feature + val featureAssignmentCount = mutable.Map[USSFeature[Long], Int]() + TweetFeatures.foreach(feature => featureAssignmentCount(feature) = 0) + + // 5. maintain a queue of category queues for round-robin processing + val categoryQueueQueue = mutable.Queue[mutable.Queue[Long]]() ++= categoryQueues + + val selectedTweets = mutable.Map[USSFeature[Long], Set[Long]]() + TweetFeatures.foreach(feature => selectedTweets(feature) = Set.empty[Long]) + + // 6. round-robin assignment of tweets to features, prioritize features with fewer assigned tweets + while (categoryQueueQueue.nonEmpty) { + val nextCategoryQueue = categoryQueueQueue.dequeue() + + if (nextCategoryQueue.nonEmpty) { + val tweetId = nextCategoryQueue.dequeue() + + // get the available features for this tweet + val availableFeatures = + tweetToFeaturesMap.getOrElse(tweetId, mutable.Set.empty[USSFeature[Long]]) + + if (availableFeatures.nonEmpty) { + // select the feature with the least assigned tweets + val selectedFeature = availableFeatures.toSeq.minBy(featureAssignmentCount(_)) + + // assign the tweet to the selected feature + selectedTweets(selectedFeature) += tweetId + featureAssignmentCount(selectedFeature) += 1 + } + + // if there are still tweets in this category, re-enqueue it + if (nextCategoryQueue.nonEmpty) { + categoryQueueQueue.enqueue(nextCategoryQueue) + } + } + } + + // update signals based on diversified tweets + val diversifiedSignals = allSignals.map { + case (feature, signalMap) => + val filteredMap = signalMap.filter { + case (tweetId, _) => selectedTweets(feature).contains(tweetId) + } + feature -> filteredMap + } + + diversifiedSignals + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val tweetIds = signalFn(query) + + if (tweetIds.isEmpty) { + val emptyFeatureMapBuilder = FeatureMapBuilder() + .add(USSGrokCategoryFeature, None) + + TweetFeatures.foreach { feature => + emptyFeatureMapBuilder.add(feature, Map.empty[Long, Seq[SignalInfo]]) + } + + Stitch.value(emptyFeatureMapBuilder.build()) + } else { + val stitchedResults = tweetIds.map { tweetId => + fetcher.fetch(tweetId).map { response => + val resultOpt = response.v + val categoryIdOpt = resultOpt.flatMap { result => + val entities = result.annotations.entities.getOrElse(Seq.empty) + val computedPairs = entities + .map(category => (category.score.getOrElse(0.0), category.qualifiedId.entityId)) + val sortedPairs = computedPairs.sortBy(-_._1) + val filteredIds = sortedPairs.map(_._2).filter(_ % 10000 == 0) + filteredIds.headOption + } + tweetId -> categoryIdOpt.map(_.toInt) + } + } + + Stitch.collect(stitchedResults).map { pairs => + val categoryMap = pairs.collect { + case (tweetId, Some(categoryId)) => + tweetId -> categoryId + }.toMap + + val featureMapBuilder = FeatureMapBuilder() + .add(USSGrokCategoryFeature, Some(categoryMap).filter(_.nonEmpty)) + + if (categoryMap.nonEmpty) { + // execute diversification and get the result + val diversifiedSignals = diversifySignalsByCategory(query, categoryMap) + + // update the original features + diversifiedSignals.foreach { + case (feature, signalMap) => + featureMapBuilder.add(feature, signalMap) + } + } else { + TweetFeatures.foreach { feature => + featureMapBuilder.add(feature, Map.empty[Long, Seq[SignalInfo]]) + } + } + + featureMapBuilder.build() + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSQueryFeatureHydrator.scala new file mode 100644 index 000000000..ceda0fa7e --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/USSQueryFeatureHydrator.scala @@ -0,0 +1,712 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.FeatureHydrationFailed +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.simclusters_v2.thriftscala.InternalId +import com.twitter.stitch.Stitch +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.Params +import com.twitter.tweet_mixer.candidate_source.uss_service.USSSignalCandidateSource +import com.twitter.tweet_mixer.feature.EntityTypes._ +import com.twitter.tweet_mixer.feature._ +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableUSSFeatureHydrator +import com.twitter.tweet_mixer.param.USSParams._ +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.tweet_mixer.utils.SignalUtils +import com.twitter.twistly.thriftscala.Sentiment +import com.twitter.usersignalservice.thriftscala.SignalType +import com.twitter.usersignalservice.{thriftscala => uss} +import com.twitter.util.Time +import com.twitter.util.logging.Logging +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Hydrate user signal store features + * + * @param ussSignalCandidateSource the uss candidate source + */ +@Singleton +class USSQueryFeatureHydrator @Inject() ( + ussSignalCandidateSource: USSSignalCandidateSource, + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] + with Logging { + + import USSQueryFeatureHydrator._ + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("USSSignals") + + private val scopedStatsReceiver = statsReceiver.scope("USSQueryFeatureHydrator") + private val signalScopedStatsReceiver = scopedStatsReceiver.scope("signal") + private val signalRequestsCount = signalScopedStatsReceiver.counter("requests") + private val allSignalCount = signalScopedStatsReceiver.counter("all") + private val timeFilterSignalCount = signalScopedStatsReceiver.counter("timeFilter") + private val vqvTimeFilterSignalCount = signalScopedStatsReceiver.counter("vqvTimeFilter") + private val negativeSentimentFilterSignalCount = + signalScopedStatsReceiver.counter("negativeSentimentFilter") + private val inUseSignalCount = signalScopedStatsReceiver.counter("inUse") + + private val negativeSignalTypes = Set[uss.SignalType]( + uss.SignalType.AccountBlock, + uss.SignalType.AccountMute, + uss.SignalType.TweetReport, + uss.SignalType.TweetDontLike, + uss.SignalType.NegativeSourceSignal, + uss.SignalType.NotificationOpenAndClickV1, + uss.SignalType.FeedbackNotrelevant, + uss.SignalType.FeedbackRelevant + ) + + private def bucketSignalStats(signals: Seq[uss.Signal], signalType: SignalType): Unit = { + val currentTimeMs = Time.now + val dayMs = 24 * 60 * 60 * 1000 + signals.foreach { signal => + signal.targetInternalId.foreach { + case InternalId.TweetId(tweetId) => + SnowflakeId.timeFromIdOpt(tweetId).foreach { time => + val ageDays = (currentTimeMs - time).inMillis / dayMs + val age = + if (ageDays <= 1) "1_day" + else if (ageDays <= 2) "2_days" + else if (ageDays <= 7) "7_days" + else if (ageDays <= 30) "30_days" + else "over_30_days" + signalScopedStatsReceiver.scope(signalType.toString).counter(age).incr() + } + case _: InternalId => + signalScopedStatsReceiver.scope(signalType.toString).counter("total").incr() + } + } + } + + override val features: Set[Feature[_, _]] = Set( + TweetFavorites, + Retweets, + TweetReplies, + TweetBookmarks, + OriginalTweets, + AccountFollows, + RepeatedProfileVisits, + TweetShares, + TweetPhotoExpands, + SearchTweetClicks, + ProfileTweetClicks, + TweetVideoOpens, + VideoViewTweets, + VideoViewVisibilityFilteredTweets, + VideoViewVisibility75FilteredTweets, + VideoViewVisibility100FilteredTweets, + VideoViewHighResolutionFilteredTweets, + AccountBlocks, + AccountMutes, + TweetReports, + TweetDontLikes, + RecentNotifications, + LowSignalUserFeature, + ImmersiveVideoViewTweets, + MediaImmersiveVideoViewTweets, + TvVideoViewTweets, + WatchTimeTweets, + ImmersiveWatchTimeTweets, + TvWatchTimeTweets, + MediaImmersiveWatchTimeTweets, + SearcherRealtimeHistory, + TweetDetailGoodClick1Min, + TweetFeedbackRelevant, + TweetFeedbackNotrelevant, + NegativeSourceSignal, + HighQualitySourceTweet, + HighQualitySourceUser + ) + + override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableUSSFeatureHydrator) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val featureMapBuilder = FeatureMapBuilder() + query.getOptionalUserId match { + case Some(userId) => + getUSSSignals(userId, query.params).map { nestedSignals => + nestedSignals.map { + case (signalType, ussSignals) => + allSignalCount.incr(ussSignals.length) + val timeFilteredSignals = ussSignals.filter { signal => + signal.timestamp >= (Time.now - Time.fromHours( + query.params(UnifiedMaxSignalAgeInHours))).inMillis + } + timeFilterSignalCount.incr(ussSignals.length - timeFilteredSignals.length) + val vqvSignals: Set[SignalType] = Set( + uss.SignalType.VideoView90dPlayback50V1, + uss.SignalType.VideoView90dQualityV1, + uss.SignalType.VideoView90dQualityV1AllSurfaces, + uss.SignalType.VideoView90dQualityV2, + uss.SignalType.VideoView90dQualityV2Visibility75, + uss.SignalType.VideoView90dQualityV2Visibility100, + uss.SignalType.VideoView90dQualityV3 + ) + val vqvTimeFilteredSignals = timeFilteredSignals.filter { signal => + (!vqvSignals.contains(signalType) && !vqvSignals.contains(signal.signalType)) || + signal.timestamp >= (Time.now - Time.fromDays( + query.params(VQVMaxSignalAgeInDays) + )).inMillis + } + vqvTimeFilterSignalCount + .incr(timeFilteredSignals.length - vqvTimeFilteredSignals.length) + val negativeSentimentFilteredSignals = vqvTimeFilteredSignals.filterNot { signal => + query.params(EnableNegativeSentimentSignalFilter) && + signal.sentiment.contains(Sentiment.Negative) + } + negativeSentimentFilterSignalCount + .incr(vqvTimeFilteredSignals.length - negativeSentimentFilteredSignals.length) + inUseSignalCount.incr(negativeSentimentFilteredSignals.length) + + // Track age stats for all signal types + if (negativeSignalTypes.contains(signalType)) + bucketSignalStats(ussSignals, signalType) + else bucketSignalStats(negativeSentimentFilteredSignals, signalType) + + signalType match { + case uss.SignalType.TweetFavorite => + featureMapBuilder.add( + TweetFavorites, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.Retweet => + featureMapBuilder.add( + Retweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.Reply => + featureMapBuilder.add( + TweetReplies, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.TweetBookmarkV1 => + featureMapBuilder.add( + TweetBookmarks, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.OriginalTweet => + featureMapBuilder.add( + OriginalTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.AccountFollow => + featureMapBuilder.add( + AccountFollows, + getSignalMap[UserId](negativeSentimentFilteredSignals)) + case uss.SignalType.RepeatedProfileVisit14dMinVisit2V1 | + uss.SignalType.RepeatedProfileVisit90dMinVisit6V1 | + uss.SignalType.RepeatedProfileVisit180dMinVisit6V1 => + featureMapBuilder.add( + RepeatedProfileVisits, + getSignalMap[UserId](negativeSentimentFilteredSignals)) + case uss.SignalType.TweetShareV1 => + featureMapBuilder.add( + TweetShares, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.TweetPhotoExpand => + featureMapBuilder.add( + TweetPhotoExpands, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.SearchTweetClick => + featureMapBuilder.add( + SearchTweetClicks, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.ProfileTweetClick => + featureMapBuilder.add( + ProfileTweetClicks, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.TweetVideoOpen => + featureMapBuilder.add( + TweetVideoOpens, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.TweetDetailGoodClick1Min => + featureMapBuilder.add( + TweetDetailGoodClick1Min, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.VideoView90dPlayback50V1 | + uss.SignalType.VideoView90dQualityV1 | + uss.SignalType.VideoView90dQualityV1AllSurfaces => + featureMapBuilder.add( + VideoViewTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.VideoView90dQualityV2 => + featureMapBuilder.add( + VideoViewVisibilityFilteredTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.VideoView90dQualityV2Visibility75 => + featureMapBuilder.add( + VideoViewVisibility75FilteredTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.VideoView90dQualityV2Visibility100 => + featureMapBuilder.add( + VideoViewVisibility100FilteredTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.VideoView90dQualityV3 => + featureMapBuilder.add( + VideoViewHighResolutionFilteredTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.ImmersiveVideoQualityView => + featureMapBuilder.add( + ImmersiveVideoViewTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.ImmersiveMediaVideoQualityView => + featureMapBuilder.add( + MediaImmersiveVideoViewTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.TvHomeVideoQualityView => + featureMapBuilder.add( + TvVideoViewTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.VideoWatchTimeAllSurfaces => + featureMapBuilder.add( + WatchTimeTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.ImmersiveVideoWatchTime => + featureMapBuilder.add( + ImmersiveWatchTimeTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.ImmersiveMediaVideoWatchTime => + featureMapBuilder.add( + MediaImmersiveWatchTimeTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.TvHomeVideoWatchTime => + featureMapBuilder.add( + TvWatchTimeTweets, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + case uss.SignalType.SearcherRealtimeHistory => + featureMapBuilder.add( + SearcherRealtimeHistory, + getSignalMap[SearchQuery](negativeSentimentFilteredSignals) + ) + case uss.SignalType.HighQualitySourceTweet => + val highQualitySourceTweetCount = negativeSentimentFilteredSignals.length + if (highQualitySourceTweetCount >= query.params( + HighQualitySourceTweetEligible) && query.params( + EnableHighQualitySourceSignalBucketFilter)) { + featureMapBuilder.add( + HighQualitySourceTweet, + getSignalMap[TweetId](negativeSentimentFilteredSignals) + ) + } + case uss.SignalType.HighQualitySourceUser => + val highQualitySourceUserCount = negativeSentimentFilteredSignals.length + if (highQualitySourceUserCount >= query.params( + HighQualitySourceUserEligible) && query.params( + EnableHighQualitySourceSignalBucketFilter)) { + featureMapBuilder.add( + HighQualitySourceUser, + getSignalMap[UserId](negativeSentimentFilteredSignals) + ) + } + // Negative signals + case uss.SignalType.AccountBlock => + featureMapBuilder.add(AccountBlocks, getSignalMap[UserId](ussSignals)) + case uss.SignalType.AccountMute => + featureMapBuilder.add(AccountMutes, getSignalMap[UserId](ussSignals)) + case uss.SignalType.TweetReport => + featureMapBuilder.add(TweetReports, getSignalMap[TweetId](ussSignals)) + case uss.SignalType.TweetDontLike => + featureMapBuilder.add(TweetDontLikes, getSignalMap[TweetId](ussSignals)) + case uss.SignalType.NegativeSourceSignal => + featureMapBuilder.add(NegativeSourceSignal, getSignalMap[TweetId](ussSignals)) + // Tweet feedback signals + // Notification-specific Signals + // Aggregated Signals + case uss.SignalType.NotificationOpenAndClickV1 => + featureMapBuilder.add(RecentNotifications, getSignalMap[TweetId](ussSignals)) + + // Survey feedback Signals + case uss.SignalType.FeedbackNotrelevant => + featureMapBuilder.add(TweetFeedbackNotrelevant, getSignalMap[TweetId](ussSignals)) + case uss.SignalType.FeedbackRelevant => + featureMapBuilder.add( + TweetFeedbackRelevant, + getSignalMap[TweetId](negativeSentimentFilteredSignals)) + // Default + case _ => () + + } + } + featureMapBuilder.add(LowSignalUserFeature, lowSignalUser(nestedSignals, query.params)) + defaultFeatureMap ++ featureMapBuilder.build() + } + case _ => Stitch.value(defaultFeatureMap) + } + } + + def lowSignalUser( + signals: Seq[(uss.SignalType, Seq[uss.Signal])], + params: Params + ): Boolean = { + val explicitSignals = signals.collect { + case (signalType, signals) if SignalUtils.ExplicitSignals.contains(signalType) => signals + }.flatten + + val timeFilteredExplicitSignals = explicitSignals.filter { signal => + getTimestamp(signal).exists { signalTime => + Time.now.since(signalTime) < params(LowSignalUserMaxSignalAge) + } + } + + timeFilteredExplicitSignals.size < params(LowSignalUserMaxSignalCount) + } + + def getUSSSignals( + userId: Long, + params: Params + ): Stitch[Seq[(uss.SignalType, Seq[uss.Signal])]] = { + val batchSignalRequest = + uss.BatchSignalRequest(userId, buildRequests(params), Some(uss.ClientIdentifier.CrMixerHome)) + + signalRequestsCount.incr() + ussSignalCandidateSource(batchSignalRequest) + } + + def buildSignalRequest( + signalType: uss.SignalType, + params: Params, + sourceSignalParam: FSParam[Boolean], + maxResultsParam: FSBoundedParam[Int] = UnifiedMaxSourceKeyNum + ): Option[uss.SignalRequest] = { + if (params(sourceSignalParam)) { + Some( + uss.SignalRequest( + maxResults = Some(params(maxResultsParam)), + signalType = signalType, + minFavCount = Some(params(UnifiedMinSignalFavCount)), + maxFavCount = Some(params(UnifiedMaxSignalFavCount)) + )) + } else None + } + + def buildRequests(params: Params): Seq[uss.SignalRequest] = { + // Tweet-based Signals + val maybeTweetFavorite = buildSignalRequest( + uss.SignalType.TweetFavorite, + params, + EnableRecentTweetFavorites, + MaxFavSignals + ) + val maybeRetweet = buildSignalRequest( + uss.SignalType.Retweet, + params, + EnableRecentTweetFavorites + ) + val maybeTweetReply = buildSignalRequest( + uss.SignalType.Reply, + params, + EnableRecentReplies + ) + val maybeTweetBookmark = buildSignalRequest( + uss.SignalType.TweetBookmarkV1, + params, + EnableRecentTweetBookmarks, + MaxBookmarkSignals + ) + val maybeOriginalTweet = buildSignalRequest( + uss.SignalType.OriginalTweet, + params, + EnableRecentOriginalTweets + ) + val maybeTweetShares = buildSignalRequest( + uss.SignalType.TweetShareV1, + params, + EnableTweetShares + ) + val maybeTweetPhotoExpand = buildSignalRequest( + uss.SignalType.TweetPhotoExpand, + params, + EnableTweetPhotoExpand, + MaxPhotoExpandSignals + ) + val maybeSearchTweetClick = buildSignalRequest( + uss.SignalType.SearchTweetClick, + params, + EnableSearchTweetClick, + MaxSearchTweetClick + ) + val maybeProfileTweetClick = buildSignalRequest( + uss.SignalType.ProfileTweetClick, + params, + EnableProfileTweetClick, + MaxProfileTweetClick + ) + val maybeTweetVideoOpen = buildSignalRequest( + uss.SignalType.TweetVideoOpen, + params, + EnableTweetVideoOpen, + MaxTweetVideoOpen + ) + val maybeTweetDetailGoodClick1Min = buildSignalRequest( + uss.SignalType.TweetDetailGoodClick1Min, + params, + EnableTweetDetailGoodClick1Min + ) + val maybeTweetFeedbackRelevant = buildSignalRequest( + uss.SignalType.FeedbackRelevant, + params, + EnableRecentTweetFeedbackRelevant + ) + val maybeTweetFeedbackNotrelevant = buildSignalRequest( + uss.SignalType.FeedbackNotrelevant, + params, + EnableRecentTweetFeedbackRelevant + ) + val maybeVideoViewTweets = buildSignalRequest( + videoViewTweetTypeParam(params(VideoViewTweetTypeParam)), + params, + EnableVideoViewTweets, + MaxVideoViewSourceSignals + ) + val maybeVideoViewVisibilityFilteredTweets = buildSignalRequest( + videoViewTweetTypeParam(params(VideoViewVisibilityFilteredTweetTypeParam)), + params, + EnableVideoViewVisibilityFilteredTweets, + MaxVqvSignals + ) + val maybeVideoViewVisibility75FilteredTweets = buildSignalRequest( + videoViewTweetTypeParam(params(VideoViewVisibility75FilteredTweetTypeParam)), + params, + EnableVideoViewVisibility75FilteredTweets, + MaxVqvSignals + ) + val maybeVideoViewVisibility100FilteredTweets = buildSignalRequest( + videoViewTweetTypeParam(params(VideoViewVisibility100FilteredTweetTypeParam)), + params, + EnableVideoViewVisibility100FilteredTweets, + MaxVqvSignals + ) + val maybeVideoViewHighResolutionFilteredTweets = buildSignalRequest( + videoViewTweetTypeParam(params(VideoViewHighResolutionFilteredTweetTypeParam)), + params, + EnableVideoViewHighResolutionFilteredTweets, + MaxVqvSignals + ) + val maybeImmersiveVideoViewTweets = buildSignalRequest( + uss.SignalType.ImmersiveVideoQualityView, + params, + EnableImmersiveVideoViewTweets + ) + val maybeMediaImmersiveVideoViewTweets = buildSignalRequest( + uss.SignalType.ImmersiveMediaVideoQualityView, + params, + EnableMediaImmersiveVideoViewTweets + ) + val maybeTvVideoViewTweets = buildSignalRequest( + uss.SignalType.TvHomeVideoQualityView, + params, + EnableTvVideoViewTweets + ) + val maybeWatchTimeTweets = buildSignalRequest( + uss.SignalType.VideoWatchTimeAllSurfaces, + params, + EnableWatchTimeTweets + ) + val maybeImmersiveWatchTimeTweets = buildSignalRequest( + uss.SignalType.ImmersiveVideoWatchTime, + params, + EnableImmersiveWatchTimeTweets + ) + val maybeMediaImmersiveWatchTimeTweets = buildSignalRequest( + uss.SignalType.ImmersiveMediaVideoWatchTime, + params, + EnableMediaImmersiveWatchTimeTweets + ) + val maybeTvWatchTimeTweets = buildSignalRequest( + uss.SignalType.TvHomeVideoWatchTime, + params, + EnableTvWatchTimeTweets + ) + val maybeSearcherRealtimeHistory = buildSignalRequest( + uss.SignalType.SearcherRealtimeHistory, + params, + EnableSearcherRealtimeHistory + ) + val maybeHighQualitySourceTweet = buildSignalRequest( + uss.SignalType.HighQualitySourceTweet, + params, + EnableHighQualitySourceTweet, + MaxHighQualitySourceSignals + ) + val maybeHighQualitySourceUser = buildSignalRequest( + uss.SignalType.HighQualitySourceUser, + params, + EnableHighQualitySourceUser, + MaxHighQualitySourceSignals + ) + + // Producer-based Signals + val maybeAccountFollow = buildSignalRequest( + uss.SignalType.AccountFollow, + params, + EnableRecentFollows + ) + val maybeRepeatedProfileVisits = buildSignalRequest( + profileMinVisitParam(params(ProfileMinVisitType)), + params, + EnableRepeatedProfileVisits + ) + + // Notification-specific Signals + val maybeRecentNotifications = buildSignalRequest( + uss.SignalType.NotificationOpenAndClickV1, + params, + EnableRecentNotifications + ) + + val enabledNegativeSignalTypes = Set( + uss.SignalType.AccountBlock, + uss.SignalType.AccountMute, + uss.SignalType.TweetReport, + uss.SignalType.TweetDontLike, + ) + + // negative signals + val maybeNegativeSignals = enabledNegativeSignalTypes + .flatMap(negativeSignal => buildSignalRequest(negativeSignal, params, EnableNegativeSignals)) + + val maybeNegativeSourceSignals = buildSignalRequest( + uss.SignalType.NegativeSourceSignal, + params, + EnableNegativeSourceSignal, + MaxNegativeSourceSignals + ) + + val allPositiveSignals = Seq( + maybeTweetFavorite, + maybeRetweet, + maybeTweetReply, + maybeTweetBookmark, + maybeOriginalTweet, + maybeAccountFollow, + maybeRepeatedProfileVisits, + maybeRecentNotifications, + maybeTweetShares, + maybeTweetPhotoExpand, + maybeSearchTweetClick, + maybeProfileTweetClick, + maybeTweetVideoOpen, + maybeTweetFeedbackRelevant, + maybeTweetDetailGoodClick1Min, + maybeVideoViewTweets, + maybeVideoViewVisibilityFilteredTweets, + maybeVideoViewVisibility75FilteredTweets, + maybeVideoViewVisibility100FilteredTweets, + maybeVideoViewHighResolutionFilteredTweets, + maybeImmersiveVideoViewTweets, + maybeMediaImmersiveVideoViewTweets, + maybeTvVideoViewTweets, + maybeWatchTimeTweets, + maybeImmersiveWatchTimeTweets, + maybeMediaImmersiveWatchTimeTweets, + maybeTvWatchTimeTweets, + maybeSearcherRealtimeHistory, + maybeHighQualitySourceTweet, + maybeHighQualitySourceUser + ) + + allPositiveSignals.flatten ++ maybeNegativeSignals ++ maybeTweetFeedbackNotrelevant ++ maybeNegativeSourceSignals + } +} + +object USSQueryFeatureHydrator { + + val defaultFeatureMap = FeatureMapBuilder() + .add(TweetFavorites, Map.empty[TweetId, Seq[SignalInfo]]) + .add(Retweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(TweetReplies, Map.empty[TweetId, Seq[SignalInfo]]) + .add(TweetBookmarks, Map.empty[TweetId, Seq[SignalInfo]]) + .add(OriginalTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(AccountFollows, Map.empty[UserId, Seq[SignalInfo]]) + .add(RepeatedProfileVisits, Map.empty[UserId, Seq[SignalInfo]]) + .add(TweetShares, Map.empty[TweetId, Seq[SignalInfo]]) + .add(TweetPhotoExpands, Map.empty[TweetId, Seq[SignalInfo]]) + .add(SearchTweetClicks, Map.empty[TweetId, Seq[SignalInfo]]) + .add(ProfileTweetClicks, Map.empty[TweetId, Seq[SignalInfo]]) + .add(TweetVideoOpens, Map.empty[TweetId, Seq[SignalInfo]]) + .add(TweetDetailGoodClick1Min, Map.empty[TweetId, Seq[SignalInfo]]) + .add(VideoViewTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(VideoViewVisibilityFilteredTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(VideoViewVisibility75FilteredTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(VideoViewVisibility100FilteredTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(VideoViewHighResolutionFilteredTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(ImmersiveVideoViewTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(MediaImmersiveVideoViewTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(TvVideoViewTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(WatchTimeTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(ImmersiveWatchTimeTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(MediaImmersiveWatchTimeTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(TvWatchTimeTweets, Map.empty[TweetId, Seq[SignalInfo]]) + .add(SearcherRealtimeHistory, Map.empty[SearchQuery, Seq[SignalInfo]]) + .add(AccountBlocks, Map.empty[UserId, Seq[SignalInfo]]) + .add(AccountMutes, Map.empty[UserId, Seq[SignalInfo]]) + .add(TweetReports, Map.empty[TweetId, Seq[SignalInfo]]) + .add(TweetDontLikes, Map.empty[TweetId, Seq[SignalInfo]]) + .add(RecentNotifications, Map.empty[TweetId, Seq[SignalInfo]]) + .add(LowSignalUserFeature, false) + .add(TweetFeedbackRelevant, Map.empty[TweetId, Seq[SignalInfo]]) + .add(TweetFeedbackNotrelevant, Map.empty[TweetId, Seq[SignalInfo]]) + .add(NegativeSourceSignal, Map.empty[TweetId, Seq[SignalInfo]]) + .add(HighQualitySourceTweet, Map.empty[TweetId, Seq[SignalInfo]]) + .add(HighQualitySourceUser, Map.empty[UserId, Seq[SignalInfo]]) + .build() + + private def getTimestamp(signal: uss.Signal): Option[Time] = { + if (signal.timestamp == 0L) None else Some(Time.fromMilliseconds(signal.timestamp)) + } + + private def extractAuthorId(signal: uss.Signal): Option[Long] = { + signal.authorId match { + case Some(InternalId.UserId(userId)) => Some(userId) + case _ => None + } + } + + def getSignalMap[T]( + ussSignals: Seq[uss.Signal] + ): Map[T, Seq[SignalInfo]] = { + ussSignals + .map { signal => + signal.targetInternalId match { + case Some(InternalId.TweetId(tweetId)) => + ( + tweetId.asInstanceOf[T], + SignalInfo( + signalEntity = uss.SignalEntity.Tweet, + signalType = signal.signalType, + sourceEventTime = getTimestamp(signal), + authorId = extractAuthorId(signal) + ) + ) + case Some(InternalId.UserId(userId)) => + ( + userId.asInstanceOf[T], + SignalInfo( + signalEntity = uss.SignalEntity.User, + signalType = signal.signalType, + sourceEventTime = getTimestamp(signal), + authorId = Some(userId) + ) + ) + case Some(InternalId.SearchQuery(searchQuery)) => + ( + searchQuery.asInstanceOf[T], + SignalInfo( + signalEntity = uss.SignalEntity.SearchQuery, + signalType = signal.signalType, + sourceEventTime = getTimestamp(signal), + authorId = None + ) + ) + case _ => throw PipelineFailure(FeatureHydrationFailed, "Unsupported Internal ID") + } + }.groupBy(_._1).mapValues(_.map(_._2)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UTGOutlierSignalsQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UTGOutlierSignalsQueryFeatureHydrator.scala new file mode 100644 index 000000000..3d3c3fb85 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UTGOutlierSignalsQueryFeatureHydrator.scala @@ -0,0 +1,94 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.param.UTGParams.OutlierFilterPercentileThresholdParam +import com.twitter.tweet_mixer.param.UTGParams.OutlierMinRequiredSignalsParam + +import javax.inject.Inject + +object UTGOutlierSignalsFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Set[Long]] { + override def defaultValue: Set[Long] = Set.empty +} + +class UTGOutlierSignalsQueryFeatureHydrator @Inject() (stats: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + + private val statsScope: StatsReceiver = stats.scope("utg_outlier_signal_counter") + private val overallFilterRate: Stat = statsScope.stat("overall_filter_rate") + private val lowSignalUserCount: Counter = statsScope.counter("low_signal_user_counter") + private val filteredCount: Counter = statsScope.counter("filtered_signal_counter") + private val totalCount: Counter = statsScope.counter("total_signal_counter") + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( + "UTGOutlierSignals") + + override def features: Set[Feature[_, _]] = Set(UTGOutlierSignalsFeature) + + private def dot(a: Array[Double], b: Array[Double]): Double = { + a.zip(b).map { case (x, y) => x * y }.sum + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val embeddingMap = + query.features.get + .getOrElse(OutlierDeepRetrievalTweetEmbeddingFeature, None).getOrElse(Map.empty) + val tweetIds = embeddingMap.keys.toSeq + + val outliers = { + if (tweetIds.length < query.params(OutlierMinRequiredSignalsParam)) { + lowSignalUserCount.incr() + Set.empty[Long] + } else { + val vectors = tweetIds.map(id => embeddingMap(id).map(_.toDouble).toArray).toArray + + val normalized = vectors.map { vec => + val norm = math.sqrt(vec.map(x => x * x).sum) + if (norm == 0.0) vec else vec.map(_ / norm) + } + + val n = normalized.length + + val dists = Array.tabulate(n, n) { (i, j) => + if (i == j) Double.PositiveInfinity else 1.0 - dot(normalized(i), normalized(j)) + } + + val minDists = dists.map(_.min) + + val sortedMinDists = minDists.sorted + + val percentileThreshold = query.params(OutlierFilterPercentileThresholdParam) / 100.0 + + val rank = percentileThreshold * (n - 1) + val lower = math.floor(rank).toInt + val upper = math.ceil(rank).toInt + val threshold = if (lower == upper) { + sortedMinDists(lower) + } else { + sortedMinDists(lower) + (rank - lower) * (sortedMinDists(upper) - sortedMinDists(lower)) + } + + val outlierSet = + tweetIds.zip(minDists).collect { case (id, dist) if dist > threshold => id }.toSet + + filteredCount.incr(outlierSet.size) + totalCount.incr(tweetIds.length) + overallFilterRate.add(outlierSet.size / tweetIds.length.toFloat) + + outlierSet + } + } + + Stitch.value( + FeatureMap(UTGOutlierSignalsFeature, outliers) + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UVGOutlierSignalsQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UVGOutlierSignalsQueryFeatureHydrator.scala new file mode 100644 index 000000000..6ffd573d5 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UVGOutlierSignalsQueryFeatureHydrator.scala @@ -0,0 +1,94 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.param.UVGParams.OutlierFilterPercentileThresholdParam +import com.twitter.tweet_mixer.param.UVGParams.OutlierMinRequiredSignalsParam + +import javax.inject.Inject + +object UVGOutlierSignalsFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Set[Long]] { + override def defaultValue: Set[Long] = Set.empty +} + +class UVGOutlierSignalsQueryFeatureHydrator @Inject() (stats: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] { + + private val statsScope: StatsReceiver = stats.scope("uvg_outlier_signal_counter") + private val overallFilterRate: Stat = statsScope.stat("overall_filter_rate") + private val lowSignalUserCount: Counter = statsScope.counter("low_signal_user_counter") + private val filteredCount: Counter = statsScope.counter("filtered_signal_counter") + private val totalCount: Counter = statsScope.counter("total_signal_counter") + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UVGOutlierSignals") + + override def features: Set[Feature[_, _]] = Set(UVGOutlierSignalsFeature) + + private def dot(a: Array[Double], b: Array[Double]): Double = { + a.zip(b).map { case (x, y) => x * y }.sum + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val embeddingMap = + query.features.get + .getOrElse(OutlierDeepRetrievalTweetEmbeddingFeature, None).getOrElse(Map.empty) + val tweetIds = embeddingMap.keys.toSeq + + val outliers = { + if (tweetIds.length < query.params(OutlierMinRequiredSignalsParam)) { + lowSignalUserCount.incr() + Set.empty[Long] + } else { + val vectors = tweetIds.map(id => embeddingMap(id).map(_.toDouble).toArray).toArray + + val normalized = vectors.map { vec => + val norm = math.sqrt(vec.map(x => x * x).sum) + if (norm == 0.0) vec else vec.map(_ / norm) + } + + val n = normalized.length + + val dists = Array.tabulate(n, n) { (i, j) => + if (i == j) Double.PositiveInfinity else 1.0 - dot(normalized(i), normalized(j)) + } + + val minDists = dists.map(_.min) + + val sortedMinDists = minDists.sorted + + val percentileThreshold = query.params(OutlierFilterPercentileThresholdParam) / 100.0 + + val rank = percentileThreshold * (n - 1) + val lower = math.floor(rank).toInt + val upper = math.ceil(rank).toInt + val threshold = if (lower == upper) { + sortedMinDists(lower) + } else { + sortedMinDists(lower) + (rank - lower) * (sortedMinDists(upper) - sortedMinDists(lower)) + } + + val outlierSet = + tweetIds.zip(minDists).collect { case (id, dist) if dist > threshold => id }.toSet + + filteredCount.incr(outlierSet.size) + totalCount.incr(tweetIds.length) + overallFilterRate.add(outlierSet.size / tweetIds.length.toFloat) + + outlierSet + } + } + + Stitch.value( + FeatureMap(UVGOutlierSignalsFeature, outliers) + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UecAggTweetTotalFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UecAggTweetTotalFeatureHydrator.scala new file mode 100644 index 000000000..f2111ebbf --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UecAggTweetTotalFeatureHydrator.scala @@ -0,0 +1,102 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.unified_counter.service.UecAggTotalOnTweetClientColumn +import com.twitter.strato.graphql.unified_counter.thriftscala.UecAggTotalRequest +import com.twitter.strato.graphql.unified_counter.thriftscala.UecAggTotalResponse +import com.twitter.strato.graphql.unified_counter.thriftscala.UecAnalyticsEngagementTypes +import com.twitter.strato.graphql.unified_counter.thriftscala.DataVersion +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines + +object UecAggTweetTotalFeature extends Feature[TweetCandidate, Option[UecAggTotalResponse]] + +class UecAggTweetTotalFeatureHydrator( + uecAggTotalOnTweetClientColumn: UecAggTotalOnTweetClientColumn, + candidatePipelinesToInclude: Option[Set[CandidatePipelineIdentifier]] = None, + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UecAggTweetTotal") + + override val features: Set[Feature[_, _]] = + Set(UecAggTweetTotalFeature) + + private val failureCounter = statsReceiver.counter("UecAggTweetTotalFeatureHydratorFailedCount") + private val skippedCandidatesCounter = + statsReceiver.counter("UecAggTweetTotalFeatureHydratorSkippedCandidates") + private val inCandidateScopeCandidatesCounter = + statsReceiver.counter("UecAggTweetTotalFeatureHydratorInCandidateScopeCandidates") + + def isInCandidateScope(features: FeatureMap): Boolean = { + candidatePipelinesToInclude + .map(candidatePipelines => + features.get(CandidatePipelines).exists(candidatePipelines.contains(_))).getOrElse(false) + } + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { + val filteredCandidates = candidates.filter { candidate => + val keepCandidate = isInCandidateScope(candidate.features) + + if (keepCandidate) inCandidateScopeCandidatesCounter.incr() + else skippedCandidatesCounter.incr() + + keepCandidate + } + + if (filteredCandidates.isEmpty) { + Stitch.value(candidates.map(_ => FeatureMap(UecAggTweetTotalFeature, None))) + } else { + val request = UecAggTotalRequest( + engagements = Some(Seq(UecAnalyticsEngagementTypes.Displayed)), + dataVersion = Some(DataVersion.ColdContentExplore), + longCache = Some(true) + ) + + val tweetIds = filteredCandidates.map(_.candidate.id).distinct + Stitch + .collect { + tweetIds.map { tweetId => + uecAggTotalOnTweetClientColumn.fetcher + .fetch(tweetId, request) + .map { response => + tweetId -> response.v + } + .handle { + case _: Throwable => + failureCounter.incr() + tweetId -> None + } + } + }.map { tweetIdsToResponseList => + val tweetIdsToResponseMap = tweetIdsToResponseList.toMap + + candidates.map { candidate => + val shouldHydrate = filteredCandidates.exists(_.candidate.id == candidate.candidate.id) + + if (shouldHydrate) { + val tweetId = candidate.candidate.id + FeatureMap( + UecAggTweetTotalFeature, + tweetIdsToResponseMap.getOrElse(tweetId, None) + ) + } else { + FeatureMap(UecAggTweetTotalFeature, None) + } + } + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserInterestSummaryQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserInterestSummaryQueryFeatureHydrator.scala new file mode 100644 index 000000000..6c7783c28 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserInterestSummaryQueryFeatureHydrator.scala @@ -0,0 +1,63 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.content_understanding.GroxUserInterestsMhClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +object UserInterestSummaryEmbeddingFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Seq[Seq[Double]]]] { + override def defaultValue: Option[Seq[Seq[Double]]] = None +} + +@Singleton +class UserInterestSummaryEmbeddingQueryFeatureHydratorFactory @Inject() ( + embeddingClientColumn: GroxUserInterestsMhClientColumn) { + + def build(): UserInterestSummaryEmbeddingQueryFeatureHydrator = { + new UserInterestSummaryEmbeddingQueryFeatureHydrator( + embeddingClientColumn + ) + } +} + +class UserInterestSummaryEmbeddingQueryFeatureHydrator( + groxUserInterestsMhClientColumn: GroxUserInterestsMhClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserInterestSummaryEmbedding") + + override val features: Set[Feature[_, _]] = Set(UserInterestSummaryEmbeddingFeature) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + getEmbedding(userId) + .map { embedding => + new FeatureMapBuilder() + .add(UserInterestSummaryEmbeddingFeature, embedding) + .build() + } + } + + private def getEmbedding(userId: Long): Stitch[Option[Seq[Seq[Double]]]] = { + groxUserInterestsMhClientColumn.fetcher + .fetch(userId).map { result => + result.v match { + case Some(userInterests) if userInterests.seedPostSummaries.isDefined => + userInterests.seedPostSummaries.map { summaries => + summaries.flatMap(_.summaryEmbedding).filter(_.nonEmpty) + } + case other => + None + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserSignalQueryFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserSignalQueryFeatureHydrator.scala new file mode 100644 index 000000000..cdc64d713 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserSignalQueryFeatureHydrator.scala @@ -0,0 +1,81 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.simclusters_v2.thriftscala.InternalId +import com.twitter.stitch.Stitch +import com.twitter.strato.client.Fetcher +import com.twitter.strato.generated.client.recommendations.user_signal_service.SignalsClientColumn +import com.twitter.usersignalservice.thriftscala.SignalType +import com.twitter.usersignalservice.{thriftscala => uss} +import com.twitter.util.Time +import javax.inject.Inject +import javax.inject.Singleton + +object UserEngagedAuthorIdsFeature + extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Seq[Long]]] { + override def defaultValue: Option[Seq[Long]] = None +} +@Singleton +case class UserSignalQueryFeatureHydrator @Inject() ( + signalsClientColumn: SignalsClientColumn) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("UserSignalQueryFeatureHydrator") + + override val features: Set[Feature[_, _]] = Set(UserEngagedAuthorIdsFeature) + + val fetcher: Fetcher[SignalsClientColumn.Key, Unit, SignalsClientColumn.Value] = + signalsClientColumn.fetcher + + val MaxFetch = 15L + val MaxSignalAge = 14.days + + private val enabledSignalTypes = Seq( + SignalType.TweetFavorite, + SignalType.Retweet, + SignalType.Reply, + SignalType.TweetBookmarkV1, + ) + + private def getTimestamp(signal: uss.Signal): Option[Time] = { + if (signal.timestamp == 0L) None else Some(Time.fromMilliseconds(signal.timestamp)) + } + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val signalRequests = enabledSignalTypes.map { signal => + uss.SignalRequest(maxResults = Some(MaxFetch), signalType = signal) + } + + val batchSignalRequest = uss.BatchSignalRequest( + userId = query.getRequiredUserId, + signalRequest = signalRequests, + clientId = Some(uss.ClientIdentifier.CrMixerHome) + ) + + fetcher.fetch(batchSignalRequest).map { response => + val signals = response.v.map(_.signalResponse.values.toSeq.flatten).getOrElse(Seq.empty) + + val timeFilteredSignals = signals.filter { signal => + getTimestamp(signal).exists { signalTime => + Time.now.since(signalTime) < MaxSignalAge + } + } + + val authorIds: Seq[Long] = timeFilteredSignals.flatMap { signal => + signal.authorId match { + case Some(InternalId.UserId(authorId)) => Some(authorId) + case _ => None + } + } + + FeatureMap(UserEngagedAuthorIdsFeature, Some(authorIds)) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserTopicIdsFeatureHydrator.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserTopicIdsFeatureHydrator.scala new file mode 100644 index 000000000..cafbda854 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator/UserTopicIdsFeatureHydrator.scala @@ -0,0 +1,137 @@ +package com.twitter.tweet_mixer.functional_component.hydrator + +import com.twitter.tweet_mixer.feature.UserTopicIdsFeature +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.interests.thriftscala.InterestId +import com.twitter.interests.thriftscala.InterestRelationship +import com.twitter.interests.thriftscala.InterestedInInterestModel +import com.twitter.interests.thriftscala.UserInterest +import com.twitter.interests.thriftscala.UserInterestData +import com.twitter.interests.thriftscala.UserInterestsResponse +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.model.common.Conditionally +import com.twitter.stitch.Stitch +import com.twitter.strato.catalog.Fetch +import com.twitter.strato.generated.client.interests.InterestedInInterestsClientColumn +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableUserTopicIdsHydrator +import com.twitter.util.logging.Logging + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UserTopicIdsFeatureHydrator @Inject() ( + interestedInInterestsClientColumn: InterestedInInterestsClientColumn, + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with Conditionally[PipelineQuery] + with Logging { + + private val scopedStats = statsReceiver.scope(getClass.getSimpleName) + private val numTopicsFetched = scopedStats.counter("numTopicsFetched") + private val numTopicsStats = scopedStats.stat("numTopicsStat") + private val emptyTopics = scopedStats.counter("emptyStats") + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserTopicIds") + + private val GenericTopics = Set( + , // Animation and Comics + , // Arts and Culture + , // Business and Finance + , // Careers + , // Digital Assets and Crypto + , // Entertainment + , // Events + , // Entertainment Industry + , // Fashion and Beauty + , // Fitness + , // Food + , // Gaming + , // Music + , // News + , // Outdoors + , // Politics + , // Science + , // Sports + , // Technology + , // Travel + ) + + override def onlyIf(query: PipelineQuery): Boolean = + query.params(EnableUserTopicIdsHydrator) + + override val features: Set[Feature[_, _]] = Set(UserTopicIdsFeature) + + private val EmptyFeatureMap = FeatureMap(UserTopicIdsFeature, Seq.empty) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + query.getOptionalUserId match { + case Some(userId) => + fetchInterestRelationships(userId) + .map(_.toSeq.flatten) + .map(extractAndSortTopics).map { topicIds => + numTopicsFetched.incr(topicIds.length) + numTopicsStats.add(topicIds.length) + FeatureMap(UserTopicIdsFeature, topicIds) + } + case _ => + emptyTopics.incr() + Stitch.value(EmptyFeatureMap) + } + } + + private def fetchInterestRelationships( + userId: Long + ): Stitch[Option[Seq[InterestRelationship]]] = { + val response: Stitch[Fetch.Result[UserInterestsResponse]] = + interestedInInterestsClientColumn.fetcher + .fetch(InterestedInInterestsClientColumn.Key(userId = userId, labels = None, None), Unit) + + response.map(_.v).map { + case Some(value) => + value.interests.interests.map { interests => + interests.collect { + case UserInterest(_, Some(interestData)) => getInterestRelationship(interestData) + }.flatten + } + case _ => None + } + } + + private def getInterestRelationship( + interestData: UserInterestData + ): Seq[InterestRelationship] = { + interestData match { + case UserInterestData.InterestedIn(interestModels) => + interestModels.collect { + case InterestedInInterestModel.ExplicitModel(model) => model + } + case _ => Nil + } + } + + private def extractAndSortTopics( + interestRelationships: Seq[InterestRelationship] + ): Seq[Long] = { + val topicsWithTimestamps = interestRelationships.flatMap { + case InterestRelationship.V1(relationshipV1) => + relationshipV1.interestId match { + case InterestId.SemanticCore(semanticCoreInterest) => + Some((semanticCoreInterest.id, relationshipV1.timestampMs)) + case _ => None + } + case _ => None + } + + val (genericTopics, subtopics) = topicsWithTimestamps + .sortBy { case (id, timestamp) => -timestamp } + .map { case (id, timestamp) => id } + .partition(GenericTopics.contains) + + subtopics ++ genericTopics + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/BUILD.bazel new file mode 100644 index 000000000..185ca2d4d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/BUILD.bazel @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/control_ai", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "snowflake/src/main/scala/com/twitter/snowflake/id", + "src/thrift/com/twitter/timelines/control_ai:timeline-control-ai-thrift-scala", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/FavoriteSelector.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/FavoriteSelector.scala new file mode 100644 index 000000000..315aff88a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/FavoriteSelector.scala @@ -0,0 +1,33 @@ +package com.twitter.tweet_mixer.functional_component.selector + +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.tweet_mixer.functional_component.hydrator.SignalInfoFeature +import com.twitter.usersignalservice.thriftscala.SignalType + +case class FavoriteSelector( + override val pipelineScope: CandidateScope) + extends Selector[PipelineQuery] { + + private def hasFavoriteSignal(candidateWithDetails: CandidateWithDetails): Boolean = { + candidateWithDetails.features + .getOrElse(SignalInfoFeature, Seq.empty) + .exists(_.signalType == SignalType.TweetFavorite) + } + + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val (favoriteCandidates, otherCandidates) = result.partition(hasFavoriteSignal) + + SelectorResult( + remainingCandidates = remainingCandidates ++ otherCandidates, + result = favoriteCandidates + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/FeedbackRelevantSelector.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/FeedbackRelevantSelector.scala new file mode 100644 index 000000000..45d8a4f58 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/FeedbackRelevantSelector.scala @@ -0,0 +1,33 @@ +package com.twitter.tweet_mixer.functional_component.selector + +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.tweet_mixer.functional_component.hydrator.SignalInfoFeature +import com.twitter.usersignalservice.thriftscala.SignalType + +case class FeedbackRelevantSelector( + override val pipelineScope: CandidateScope) + extends Selector[PipelineQuery] { + + private def hasFeedbackRelevantSignal(candidateWithDetails: CandidateWithDetails): Boolean = { + candidateWithDetails.features + .getOrElse(SignalInfoFeature, Seq.empty) + .exists(_.signalType == SignalType.FeedbackRelevant) + } + + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val (feedbackRelevantCandidates, otherCandidates) = result.partition(hasFeedbackRelevantSignal) + + SelectorResult( + remainingCandidates = remainingCandidates ++ otherCandidates, + result = feedbackRelevantCandidates + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/HydraBasedSorterProvider.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/HydraBasedSorterProvider.scala new file mode 100644 index 000000000..b08d14a4b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/HydraBasedSorterProvider.scala @@ -0,0 +1,75 @@ +package com.twitter.tweet_mixer.functional_component.selector + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.selector.sorter.SorterFromOrdering +import com.twitter.product_mixer.component_library.selector.sorter.SorterProvider +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.HasVideo +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.IsLongVideo +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.isFeatureSet +import com.twitter.tweet_mixer.feature.HydraScoreFeature +import com.twitter.tweet_mixer.feature.TweetBooleanInfoFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.VideoScoreWeightParam + +object HydraBasedSorterProvider extends SorterProvider { + + // Should actually do this sort of stuff in hydra (YQian mentioned this sometime ago, save features in hydra on which scoring depends) + def longVideo(features: FeatureMap): Boolean = { + isFeatureSet(IsLongVideo, features.getOrElse(TweetBooleanInfoFeature, None).getOrElse(0)) + } + + def hasVideo(features: FeatureMap): Boolean = { + isFeatureSet(HasVideo, features.getOrElse(TweetBooleanInfoFeature, None).getOrElse(0)) + } + + def bool2int(b: Boolean) = if (b) 1 else 0 + + def sigmoid(x: Double): Double = { + if (x < -45) { + 0.0 + } else if (x > 45) { + 1.0 + } else { + 1.0 / (1.0 + math.exp(-x)) + } + } + + val IS_SELECTED_PREDS = "is_selected_preds" + val IS_VIDEO_PREDS = "is_video_preds" + + override def sorter( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SorterFromOrdering = { + + val video_score = query.params(VideoScoreWeightParam) + SorterFromOrdering( + Ordering + .by[CandidateWithDetails, Double] { + case ItemCandidateWithDetails(candidate: TweetCandidate, _, features) => + val isLongVideo = bool2int(longVideo(features)) + val isVideo = bool2int(hasVideo(features)) + val hydraScoreMap = features.getOrElse(HydraScoreFeature, Map.empty[String, Double]) + val hydraScore = hydraScoreMap match { + case scores if scores.isEmpty => Double.NegativeInfinity + case scores if scores.size == 1 => + sigmoid(scores.values.sum) * (1 + video_score * isVideo) + case scores if scores.size == 2 => + scores.getOrElse( + IS_SELECTED_PREDS, + Double.NegativeInfinity) + video_score * isLongVideo * scores + .getOrElse(IS_VIDEO_PREDS, Double.NegativeInfinity) + case _ => throw PipelineFailure(IllegalStateFailure, "Not expected") + } + hydraScore + case _ => throw PipelineFailure(IllegalStateFailure, "Unexpected candidate type") + }.reverse + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/HydraBasedTransformedSorterProvider.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/HydraBasedTransformedSorterProvider.scala new file mode 100644 index 000000000..9c4ce0f1c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/HydraBasedTransformedSorterProvider.scala @@ -0,0 +1,152 @@ +package com.twitter.tweet_mixer.functional_component.selector + +import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.UserControlAiFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.selector.sorter.SorterFromTransformedOrdering +import com.twitter.product_mixer.component_library.selector.sorter.SorterProvider +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.timelines.control_ai.control.{thriftscala => ci} +import com.twitter.tweet_mixer.candidate_pipeline.ControlAiTopicCandidatePipelineConfig +import com.twitter.tweet_mixer.feature.AuthorIdFeature +import com.twitter.tweet_mixer.feature.HydraScoreFeature +import com.twitter.tweet_mixer.feature.TweetBooleanInfoFeature +import com.twitter.tweet_mixer.feature.TweetInfoFeatures +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.isFeatureSet +import com.twitter.tweet_mixer.functional_component.transformer.ReplyFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ControlAiEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ControlAiShowLessWeightParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ControlAiShowMoreWeightParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ReplyScoreWeightParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.VideoScoreWeightParam +import scala.collection.immutable.ListSet + +private case class ControlAiConfig( + actions: Seq[ci.Action], + actionTypeWeights: Map[ci.ActionType, Double]) + +object HydraBasedTransformedSorterProvider extends SorterProvider { + + private def hasBooleanFeature(features: FeatureMap, feature: String): Boolean = { + isFeatureSet(feature, features.getOrElse(TweetBooleanInfoFeature, None).getOrElse(0)) + } + + private def bool2int(b: Boolean) = if (b) 1 else 0 + + private def sigmoid(x: Double): Double = { + if (x < -45) { + 0.0 + } else if (x > 45) { + 1.0 + } else { + 1.0 / (1.0 + math.exp(-x)) + } + } + + private val IS_SELECTED_PREDS = "is_selected_preds" + private val IS_VIDEO_PREDS = "is_video_preds" + + private def isControlAiMatch( + action: ci.Action, + candidate: TweetCandidate, + features: FeatureMap + ): Boolean = { + val conditions: Seq[ci.Condition => Boolean] = + Seq( + _.postTopic.forall { topic => + features + .getOrElse(CandidatePipelines, ListSet.empty[CandidatePipelineIdentifier]) + .contains(ControlAiTopicCandidatePipelineConfig.Identifier) + }, + _.postHasVideo.forall(_ == hasBooleanFeature(features, TweetInfoFeatures.HasVideo)), + _.postHasImage.forall(_ == hasBooleanFeature(features, TweetInfoFeatures.HasImage)), + _.postIsReply.forall(_ == hasBooleanFeature(features, TweetInfoFeatures.IsReply)), + _.postIsRetweet.forall(_ == hasBooleanFeature(features, TweetInfoFeatures.IsRetweet)), + _.postMaximumAge.forall(maxAge => + SnowflakeId.timeFromIdOpt(candidate.id).exists(_.untilNow.inMinutes <= maxAge)), + _.userFollowsAuthor.forall( + _ == false + ), + _.authorId.forall(aid => features.getOrElse(AuthorIdFeature, None).contains(aid)), + ) + + conditions.forall(_(action.condition)) + } + + private def getHydraScore( + candidate: CandidateWithDetails, + controlAiConfig: Option[ControlAiConfig], + videoWeight: Double, + replyWeight: Double, + ): Double = { + candidate match { + case ItemCandidateWithDetails(candidate: TweetCandidate, _, features) => + val isLongVideo = bool2int(hasBooleanFeature(features, TweetInfoFeatures.IsLongVideo)) + val isVideo = bool2int(hasBooleanFeature(features, TweetInfoFeatures.HasVideo)) + val isReply = bool2int(features.getOrElse(ReplyFeature, false)) + + val controlAiWeight = controlAiConfig + .flatMap { config => + val lastMatchedAction = config.actions + .filter(isControlAiMatch(_, candidate, features)).lastOption + lastMatchedAction + .map(action => config.actionTypeWeights.getOrElse(action.actionType, 1.0)) + }.getOrElse(0.0) + + val hydraScoreMap = features.getOrElse(HydraScoreFeature, Map.empty[String, Double]) + val hydraScore = hydraScoreMap match { + case scores if scores.isEmpty => Double.NegativeInfinity + case scores if scores.size == 1 => + sigmoid(scores.values.sum) * + (1 + videoWeight * isVideo) * (1 + replyWeight * isReply) * (1 + controlAiWeight) + case scores if scores.size == 2 => + scores.getOrElse( + IS_SELECTED_PREDS, + Double.NegativeInfinity) + videoWeight * isLongVideo * scores + .getOrElse(IS_VIDEO_PREDS, Double.NegativeInfinity) + case _ => throw PipelineFailure(IllegalStateFailure, "Not expected") + } + hydraScore + case _ => throw PipelineFailure(IllegalStateFailure, "Unexpected candidate type") + } + } + + override def sorter( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SorterFromTransformedOrdering[Double] = { + val videoScore = query.params(VideoScoreWeightParam) + val replyScore = query.params(ReplyScoreWeightParam) + + val controlAiConfig = { + if (query.params(ControlAiEnabled)) { + val actions = query.features + .flatMap(_.getOrElse(UserControlAiFeature, None)).map(_.actions).getOrElse(Seq.empty) + Some( + ControlAiConfig( + actions, + Map( + ci.ActionType.More -> query.params(ControlAiShowMoreWeightParam), + ci.ActionType.Less -> query.params(ControlAiShowLessWeightParam), + ci.ActionType.Only -> 10000, + ci.ActionType.Exclude -> -10000, + ) + )) + } else + None + } + + SorterFromTransformedOrdering( + Ordering.Double.reverse, + getHydraScore(_, controlAiConfig, videoScore, replyScore) + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/InsertAppendWeightedSignalPriorityWeaveResults.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/InsertAppendWeightedSignalPriorityWeaveResults.scala new file mode 100644 index 000000000..d757503df --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/InsertAppendWeightedSignalPriorityWeaveResults.scala @@ -0,0 +1,153 @@ +package com.twitter.product_mixer.component_library.selector + +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline +import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.usersignalservice.thriftscala.SignalType +import scala.annotation.tailrec +import scala.collection.mutable +import scala.util.Random + +object InsertAppendWeightedSignalPriorityWeaveResults { + def apply[Query <: PipelineQuery, Bucket]( + candidatePipelines: Set[CandidatePipelineIdentifier], + bucketer: Bucketer[Bucket], + weights: Bucket => (Option[SignalType], Double), + random: Random + ): InsertAppendWeightedSignalPriorityWeaveResults[Query, Bucket] = + new InsertAppendWeightedSignalPriorityWeaveResults( + SpecificPipelines(candidatePipelines), + bucketer, + weights, + random + ) + + def apply[Query <: PipelineQuery, Bucket]( + candidatePipeline: CandidatePipelineIdentifier, + bucketer: Bucketer[Bucket], + weights: Bucket => (Option[SignalType], Double), + random: Random + ): InsertAppendWeightedSignalPriorityWeaveResults[Query, Bucket] = + new InsertAppendWeightedSignalPriorityWeaveResults( + SpecificPipeline(candidatePipeline), + bucketer, + weights, + random + ) +} + +case class InsertAppendWeightedSignalPriorityWeaveResults[-Query <: PipelineQuery, Bucket]( + override val pipelineScope: CandidateScope, + bucketer: Bucketer[Bucket], + weights: Bucket => (Option[SignalType], Double), + random: Random) + extends Selector[Query] { + private sealed trait PatternResult + private case object NotASelectedCandidatePipeline extends PatternResult + private case class Bucketed(bucket: Bucket) extends PatternResult + + override def apply( + query: Query, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + + val groupedCandidates: Map[PatternResult, Seq[CandidateWithDetails]] = + remainingCandidates.groupBy { candidateWithDetails => + if (pipelineScope.contains(candidateWithDetails)) { + val bucket = bucketer(candidateWithDetails) + Bucketed(bucket) + } else { + NotASelectedCandidatePipeline + } + } + + val otherCandidates = + groupedCandidates.getOrElse(NotASelectedCandidatePipeline, Seq.empty) + + val groupedCandidatesIterators = groupedCandidates.collect { + case (Bucketed(bucket), candidatesWithDetails) => (bucket, candidatesWithDetails.iterator) + } + + // Group buckets by signal type to calculate normalized weights + val bucketsBySignalType = groupedCandidatesIterators.keys + .map { bucket => + val (signalTypeOpt, weight) = weights(bucket) + (bucket, signalTypeOpt, weight) + }.groupBy(_._2) + + val normalizedBucketWeights = bucketsBySignalType.flatMap { + case (signalTypeOpt, bucketsWithSignal) => + val buckets = bucketsWithSignal.map(_._1) + val normalizedWeight = if (buckets.nonEmpty) { + val weight = bucketsWithSignal.head._3 + weight / buckets.size + } else 0.0 + buckets.map(bucket => (bucket, normalizedWeight)) + } + + val currentBucketWeights: mutable.Map[Bucket, Double] = { + val bucketsAndWeightsSortedByWeight = + normalizedBucketWeights.toSeq.sortBy(_._2)(Ordering.Double.reverse) + mutable.LinkedHashMap(bucketsAndWeightsSortedByWeight: _*) + } + + var weightSum = currentBucketWeights.valuesIterator.sum + + // Add candidates to `newResults` until all remaining candidates are for a single bucket + val newResult = new mutable.ArrayBuffer[CandidateWithDetails]() + while (currentBucketWeights.size > 1) { + // Random number between 0 and the sum of the ratios of all params + val randomValue = random.nextDouble() * weightSum + + val currentBucketWeightsIterator: Iterator[(Bucket, Double)] = + currentBucketWeights.iterator + val (currentBucket, weight) = currentBucketWeightsIterator.next() + + val componentToTakeFrom = findBucketToTakeFrom( + randomValue = randomValue, + cumulativeSumOfWeights = weight, + bucket = currentBucket, + bucketWeightsIterator = currentBucketWeightsIterator + ) + + groupedCandidatesIterators.get(componentToTakeFrom) match { + case Some(iteratorForBucket) if iteratorForBucket.nonEmpty => + newResult += iteratorForBucket.next() + case _ => + weightSum -= currentBucketWeights(componentToTakeFrom) + currentBucketWeights.remove(componentToTakeFrom) + } + } + + val remainingBucketInRatio = + currentBucketWeights.keysIterator.flatMap(groupedCandidatesIterators.get).flatten + + SelectorResult( + remainingCandidates = otherCandidates, + result = result ++ newResult ++ remainingBucketInRatio) + } + + @tailrec private def findBucketToTakeFrom( + randomValue: Double, + cumulativeSumOfWeights: Double, + bucket: Bucket, + bucketWeightsIterator: Iterator[(Bucket, Double)] + ): Bucket = { + if (randomValue < cumulativeSumOfWeights || bucketWeightsIterator.isEmpty) { + bucket + } else { + val (nextBucket, weight) = bucketWeightsIterator.next() + findBucketToTakeFrom( + randomValue, + cumulativeSumOfWeights + weight, + nextBucket, + bucketWeightsIterator) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/ReserveVideoSelector.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/ReserveVideoSelector.scala new file mode 100644 index 000000000..bcd08a108 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/ReserveVideoSelector.scala @@ -0,0 +1,40 @@ +package com.twitter.tweet_mixer.functional_component.selector + +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.common.AllPipelines +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.Param +import com.twitter.tweet_mixer.feature.TweetBooleanInfoFeature +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.HasVideo +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.isFeatureSet + +case class ReserveVideoSelector( + override val pipelineScope: CandidateScope = AllPipelines, + maxResults: Param[Int]) + extends Selector[PipelineQuery] { + + def isVideo(candidateWithDetails: CandidateWithDetails): Boolean = { + val tweetBooleanInfo = + candidateWithDetails.features.getOrElse(TweetBooleanInfoFeature, None).getOrElse(0) + isFeatureSet(HasVideo, tweetBooleanInfo) + } + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val topK = query.requestedMaxResults.getOrElse(query.params(maxResults)) + val topKResult = result.take(topK) + + val (videoResultTopK, notVideoResultTopK) = topKResult.partition(isVideo) + + val (videoResult, notVideoResult) = result.partition(isVideo) + + SelectorResult( + remainingCandidates = remainingCandidates ++ notVideoResult, + result = videoResultTopK) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/UprankVideoSorterProvider.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/UprankVideoSorterProvider.scala new file mode 100644 index 000000000..e4c1a7133 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector/UprankVideoSorterProvider.scala @@ -0,0 +1,34 @@ +package com.twitter.tweet_mixer.functional_component.selector + +import com.twitter.product_mixer.component_library.selector.sorter.SorterFromOrdering +import com.twitter.product_mixer.component_library.selector.sorter.SorterProvider +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.tweet_mixer.feature.ScoreFeature +import com.twitter.tweet_mixer.feature.TweetBooleanInfoFeature +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.HasVideo +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.isFeatureSet + +object UprankVideoSorterProvider extends SorterProvider { + + private val VideoWeight = 1.8 + + override def sorter( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SorterFromOrdering = SorterFromOrdering( + Ordering.by[CandidateWithDetails, Double] { + case ItemCandidateWithDetails(_, _, features) => + val tweetBooleanInfo = features.getOrElse(TweetBooleanInfoFeature, None).getOrElse(0) + val hasVideo = isFeatureSet(HasVideo, tweetBooleanInfo) + val score = features.getOrElse(ScoreFeature, 0.0) + val updatedScore = if (hasVideo) score * VideoWeight else score + -updatedScore + case _ => throw PipelineFailure(IllegalStateFailure, "Unexpected candidate type") + } + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/BUILD.bazel new file mode 100644 index 000000000..30c4d00ec --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/BUILD.bazel @@ -0,0 +1,31 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "hydra/common/libraries/src/main/scala/com/twitter/hydra/common/utils", + "hydra/embedding-generation/thrift/src/main/thrift/com/twitter/hydra/embedding_generation:thrift-scala", + "hydra/root/thrift/src/main/thrift:thrift-scala", + "kafka/finagle-kafka/finatra-kafka/src/main/scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", + "strato/config/columns/hydra:hydra-strato-client", + "strato/config/src/thrift/com/twitter/strato/columns/content_understanding:content_understanding-scala", + "timelines/ml:kafka", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + "vecdb/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/DeepRetrievalAdHocSideEffect.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/DeepRetrievalAdHocSideEffect.scala new file mode 100644 index 000000000..b4827e18d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/DeepRetrievalAdHocSideEffect.scala @@ -0,0 +1,146 @@ +package com.twitter.tweet_mixer.functional_component.side_effect + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hydra.common.utils.{Utils => HydraUtils} +import com.twitter.hydra.embedding_generation.{thriftscala => eg} +import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller +import com.twitter.vecdb.{thriftscala => t} +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.model.response.TweetMixerResponse +import com.twitter.tweet_mixer.module.GPURetrievalHttpClient +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableDeepRetrievalAdhocDecider +import com.twitter.tweet_mixer.utils.PipelineFailureCategories.FailedEmbeddingHydrationResponse +import com.twitter.tweet_mixer.utils.PipelineFailureCategories.InvalidEmbeddingHydrationResponse +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Named +import javax.inject.Singleton + +/** + * Side effect that calls deep retrieval and GPU retrieval and compares both + */ +@Singleton +class DeepRetrievalAdHocSideEffect @Inject() ( + egsClient: eg.EmbeddingGenerationService.MethodPerEndpoint, + @Named(ModuleNames.VecDBAnnServiceClient) + annClient: t.VecDB.MethodPerEndpoint, + @Named(ModuleNames.GPURetrievalDevelHttpClient) + gpuRetrievalHttpClient: GPURetrievalHttpClient, + statsReceiver: StatsReceiver) + extends PipelineResultSideEffect[PipelineQuery, TweetMixerResponse] + with PipelineResultSideEffect.Conditionally[ + PipelineQuery, + TweetMixerResponse + ] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("DeepRetrievalAdhoc") + + private val scopedStatsReceiver = statsReceiver.scope(identifier.toString) + private val vecdbResCounter = scopedStatsReceiver.counter("vecDbResSize") + private val gpuResCounter = scopedStatsReceiver.counter("gpuResSize") + private val commonCounter = scopedStatsReceiver.counter("commonResSize") + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: TweetMixerResponse + ): Boolean = query.params(EnableDeepRetrievalAdhocDecider) + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, TweetMixerResponse] + ): Stitch[Unit] = { + val query = inputs.query + getEmbedding(query.getRequiredUserId, query.clientContext, "deep_retrieval_exp4") + .flatMap { embeddingOpt => + embeddingOpt match { + case Some(embedding) => + val vecdbFut = annClient + .search( + dataset = "tweet-deep-retrieval-exp4", + vector = HydraUtils.intBitsSeqToFloatSeq(embedding).map(_.toDouble), + params = Some(t.SearchParams(limit = Some(1000))) + ).map { response: t.SearchResponse => + response.points match { + case points: Seq[t.ScoredPoint] => + points.map { point => (point.id, point.score) } + case _ => + Seq.empty + } + } + val gpuFut = gpuRetrievalHttpClient.getNeighbors(embedding) + Stitch.callFuture { + Future.join(vecdbFut, gpuFut).map { + case (vecdbRes, gpuRes) => + val vecdbTweets = vecdbRes.map(_._1) + val gpuTweets = gpuRes.map(_._1) + vecdbResCounter.incr(vecdbTweets.size) + gpuResCounter.incr(gpuTweets.size) + commonCounter.incr(vecdbTweets.toSet.intersect(gpuTweets.toSet).size) + } + } + case None => Stitch.Unit + } + }.flatMap { _ => Stitch.Unit } + } + + private val InvalidResponseException = Stitch.exception( + PipelineFailure(InvalidEmbeddingHydrationResponse, "Invalid embedding hydration response")) + + private val FailedResponseException = Stitch.exception( + PipelineFailure(FailedEmbeddingHydrationResponse, "Failed embedding hydration response")) + + /** + * The thrift response has been designed to support multiple embeddings for different model configs. For now + * we only expect to use a single config, so we will take the first one and ignore anything else in the response + */ + private object FirstResultInResponse { + def unapply(uer: eg.UserEmbeddingsResponse): Option[eg.UserEmbeddingsResult] = + uer.results.flatMap(_.values.headOption) + } + + private def getEmbedding( + userId: Long, + clientContext: ClientContext, + modelName: String + ): Stitch[Option[Seq[Int]]] = { + val egsQuery = eg.EmbeddingsGenerationRequest( + clientContext = ClientContextMarshaller(clientContext), + product = eg.Product.UserEmbeddings, + productContext = Some( + eg.ProductContext.UserEmbeddingsContext( + eg.UserEmbeddingsContext(userIds = Seq(userId), modelNames = Some(Seq(modelName))))) + ) + + Stitch.callFuture(egsClient.generateEmbeddings(egsQuery)).flatMap { + case eg.EmbeddingsGenerationResponse + .UserEmbeddingsResponse(FirstResultInResponse(userEmbeddingsResult)) => + userEmbeddingsResult match { + case eg.UserEmbeddingsResult.UserEmbeddings(embeddings) => + // We expect the embeddings to be keyed on the user id that we passed + embeddings.embeddingsByUserId.flatMap(_.get(userId)) match { + case Some(embeddings) => + Stitch.value(Some(embeddings)) + case _ => Stitch.value(None) + } + case eg.UserEmbeddingsResult.ValidationError(error) => + Stitch.exception( + PipelineFailure( + InvalidEmbeddingHydrationResponse, + error.msg.getOrElse("Unknown validation error in EmbeddingsGenerationService") + ) + ) + case _ => InvalidResponseException + } + case _ => FailedResponseException + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/EvergreenVideosSideEffect.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/EvergreenVideosSideEffect.scala new file mode 100644 index 000000000..367cee171 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/EvergreenVideosSideEffect.scala @@ -0,0 +1,59 @@ +package com.twitter.tweet_mixer.functional_component.side_effect + +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.HistoricalEvergreenVideosCandidateSource +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.EvergreenVideosSearchByUserIdsQuery +import com.twitter.tweet_mixer.functional_component.hydrator.SGSFollowedUsersFeature +import com.twitter.tweet_mixer.model.response.TweetMixerResponse +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableEvergreenVideosSideEffect +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenVideosMaxFollowUsersParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenVideosPaginationNumParam +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Side effect that calls evergreen_videos dark traffic to fetch tweets. + */ + +@Singleton +class EvergreenVideosSideEffect @Inject() ( + evergreenVideosCandidateSource: HistoricalEvergreenVideosCandidateSource) + extends PipelineResultSideEffect[PipelineQuery, TweetMixerResponse] + with PipelineResultSideEffect.Conditionally[ + PipelineQuery, + TweetMixerResponse + ] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("EvergreenVideos") + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: TweetMixerResponse + ): Boolean = query.params(EnableEvergreenVideosSideEffect) + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, TweetMixerResponse] + ): Stitch[Unit] = { + val query = inputs.query + val paginationNum = query.params(EvergreenVideosPaginationNumParam) + val maxFollowUsers = query.params(EvergreenVideosMaxFollowUsersParam) + + val followedUserIds = query.features + .map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty[Long]) ++ Seq(query.getRequiredUserId)) + .map(_.take(maxFollowUsers)) + + val request = EvergreenVideosSearchByUserIdsQuery( + userIds = followedUserIds.getOrElse(Seq.empty), + size = paginationNum + ) + Stitch.run(evergreenVideosCandidateSource.apply(request)) + Stitch.Unit + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/HydraScoringSideEffect.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/HydraScoringSideEffect.scala new file mode 100644 index 000000000..8d97cf8ba --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/HydraScoringSideEffect.scala @@ -0,0 +1,76 @@ +package com.twitter.tweet_mixer.functional_component.side_effect + +import com.twitter.hydra.root.{thriftscala => t} +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.thriftscala.ClientContext +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.AuthorIdFeature +import com.twitter.tweet_mixer.model.response.TweetMixerResponse +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EnableHydraScoringSideEffect +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.HydraModelName +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Side effect that calls hydra dark traffic to score tweets. + */ +@Singleton +class HydraScoringSideEffect @Inject() ( + hydraRootService: t.HydraRoot.MethodPerEndpoint, +) extends PipelineResultSideEffect[PipelineQuery, TweetMixerResponse] + with PipelineResultSideEffect.Conditionally[ + PipelineQuery, + TweetMixerResponse + ] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("HydraScoring") + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: TweetMixerResponse + ): Boolean = query.params(EnableHydraScoringSideEffect) + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, TweetMixerResponse] + ): Stitch[Unit] = { + val query = inputs.query + val candidates = + inputs.selectedCandidates ++ inputs.remainingCandidates ++ inputs.droppedCandidates + val request = hydraRequest(query, candidates) + Stitch + .callFuture(hydraRootService.getRecommendationResponse(request)) + .flatMap(_ => Stitch.Unit) + } + + private def hydraRequest( + query: PipelineQuery, + candidates: Seq[CandidateWithDetails] + ): t.HydraRootRequest = { + val modelName = query.params(HydraModelName) + val tweetCandidates = candidates.map { candidate => + t.TweetCandidate( + id = candidate.candidateIdLong, + authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(-1) + ) + } + + t.HydraRootRequest( + clientContext = ClientContext(userId = query.clientContext.userId), + product = t.Product.TweetRanking, + productContext = Some( + t.ProductContext.TweetRanking( + t.TweetRanking( + candidates = tweetCandidates, + modelName = modelName + ) + ) + ) + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/PublishGroxUserInterestsSideEffect.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/PublishGroxUserInterestsSideEffect.scala new file mode 100644 index 000000000..a6ef70bd7 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/PublishGroxUserInterestsSideEffect.scala @@ -0,0 +1,49 @@ +package com.twitter.tweet_mixer.functional_component.side_effect + +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.hydra.PublishGroxUserInterestsRequestKafkaClientColumn +import com.twitter.strato.columns.content_understanding.content_exploration.thriftscala.UserInterestsRequests +import com.twitter.strato.columns.content_understanding.content_exploration.thriftscala.UserInterestsTriggerScene +import com.twitter.tweet_mixer.functional_component.hydrator.UserInterestSummaryEmbeddingFeature +import com.twitter.tweet_mixer.model.response.TweetMixerResponse +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UserInterestSummarySimilarityEnabled +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class PublishGroxUserInterestsSideEffect @Inject() ( + publishGroxUserInterestsRequestKafkaClientColumn: PublishGroxUserInterestsRequestKafkaClientColumn) + extends PipelineResultSideEffect[PipelineQuery, TweetMixerResponse] + with PipelineResultSideEffect.Conditionally[PipelineQuery, TweetMixerResponse] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("PublishGroxUserInterests") + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: TweetMixerResponse + ): Boolean = { + query.params(UserInterestSummarySimilarityEnabled) && + query.features.flatMap(_.getOrElse(UserInterestSummaryEmbeddingFeature, None)).isEmpty + } + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, TweetMixerResponse] + ): Stitch[Unit] = { + val userId = inputs.query.getRequiredUserId + val publishStitch = publishGroxUserInterestsRequestKafkaClientColumn.inserter.insert( + userId, + UserInterestsRequests( + userId = userId, + scene = Some(UserInterestsTriggerScene.TweetMixer) + ) + ) + publishStitch.map(_ => ()) + } +} \ No newline at end of file diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/RequestMultimodalEmbeddingSideEffect.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/RequestMultimodalEmbeddingSideEffect.scala new file mode 100644 index 000000000..cda7668e4 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/RequestMultimodalEmbeddingSideEffect.scala @@ -0,0 +1,65 @@ +package com.twitter.tweet_mixer.functional_component.side_effect + +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.hydra.PublishGroxMultimodalEmbedRequestKafkaClientColumn +import com.twitter.strato.columns.content_understanding.content_exploration.thriftscala.MultimodalEmbeddingRequest +import com.twitter.tweet_mixer.functional_component.hydrator.MultimodalEmbeddingFeature +import com.twitter.tweet_mixer.model.response.TweetMixerResponse +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationMultimodalEnabled +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class RequestMultimodalEmbeddingSideEffect @Inject() ( + publishGroxMultimodalEmbedRequestKafkaClientColumn: PublishGroxMultimodalEmbedRequestKafkaClientColumn) + extends PipelineResultSideEffect[PipelineQuery, TweetMixerResponse] + with PipelineResultSideEffect.Conditionally[PipelineQuery, TweetMixerResponse] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("RequestMultimodalEmbedding") + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: TweetMixerResponse + ): Boolean = { + query.params(ContentExplorationMultimodalEnabled) && { + query.features.flatMap(_.getOrElse(MultimodalEmbeddingFeature, None)) match { + case Some(embeddings) => + embeddings.values.exists(_.isEmpty) // Check if any embedding is None/missing + case None => false + } + } + } + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, TweetMixerResponse] + ): Stitch[Unit] = { + val maybeEmbeddings = + inputs.query.features.flatMap(_.getOrElse(MultimodalEmbeddingFeature, None)) + + val missingPostIds = maybeEmbeddings match { + case Some(embeddings) => + embeddings.filter { case (_, embeddingOpt) => embeddingOpt.isEmpty }.keys.toSeq + case None => Seq.empty[Long] + } + + if (missingPostIds.nonEmpty) { + // Parallelize Kafka insertions for each missing post ID (one per request) + val publishStitches = missingPostIds.map { postId => + publishGroxMultimodalEmbedRequestKafkaClientColumn.inserter.insert( + postId, + MultimodalEmbeddingRequest(postId = postId) + ) + } + + // Execute all insertions in parallel and collect results + Stitch.traverse(publishStitches) { stitch => stitch }.map(_ => ()) + } else Stitch.Unit // No missing embeddings, no need to publish + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/ScribeServedCandidatesSideEffect.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/ScribeServedCandidatesSideEffect.scala new file mode 100644 index 000000000..4bbc6b4fc --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/ScribeServedCandidatesSideEffect.scala @@ -0,0 +1,118 @@ +package com.twitter.tweet_mixer.functional_component.side_effect + +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.HasMarshalling +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.ml.kafka.serde.ThriftStructSafeSerde +import com.twitter.tweet_mixer.feature.PredictionRequestIdFeature +import com.twitter.tweet_mixer.feature.ScoreFeature +import com.twitter.tweet_mixer.feature.SourceSignalFeature +import com.twitter.tweet_mixer.functional_component.hydrator.SignalInfoFeature +import com.twitter.tweet_mixer.model.request.HomeRecommendedTweetsProduct +import com.twitter.tweet_mixer.model.request.VideoRecommendedTweetsProduct +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ScribeRetrievedCandidatesParam +import com.twitter.tweet_mixer.thriftscala.CandidateInfo +import com.twitter.tweet_mixer.thriftscala.Product +import com.twitter.tweet_mixer.thriftscala.RequestInfo +import com.twitter.tweet_mixer.thriftscala.SignalInfo +import com.twitter.tweet_mixer.thriftscala.ServedCandidatesForRequest +import com.twitter.tweet_mixer.utils.CandidateSourceUtil +import javax.inject.Inject +import javax.inject.Singleton +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.serialization.Serializer + +@Singleton +class ScribeServedCandidatesSideEffectFactory @Inject() ( + injectedServiceIdentifier: ServiceIdentifier) { + def build(identifierPrefix: String): ScribeServedCandidatesSideEffect = + ScribeServedCandidatesSideEffect(injectedServiceIdentifier, identifierPrefix) +} + +case class ScribeServedCandidatesSideEffect( + serviceIdentifier: ServiceIdentifier, + identifierPrefix: String) + extends KafkaPublishingSideEffect[ + RequestInfo, + ServedCandidatesForRequest, + PipelineQuery, + HasMarshalling + ] + with Conditionally[PipelineQuery, HasMarshalling] { + + override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" + private val kafkaTopic: String = "tweet_mixer_retrieved_candidates" + override val keySerde: Serializer[RequestInfo] = + ThriftStructSafeSerde.Thrift[RequestInfo]().serializer + override val valueSerde: Serializer[ServedCandidatesForRequest] = + ThriftStructSafeSerde.Thrift[ServedCandidatesForRequest]().serializer + override val clientId: String = "tweet_mixer_served_candidate_producer" + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("ScribeServedCandidates") + + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Boolean = { + serviceIdentifier.environment.toLowerCase == "prod" && + query.params(ScribeRetrievedCandidatesParam) && + selectedCandidates.nonEmpty + } + + override def buildRecords( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: HasMarshalling + ): Seq[ProducerRecord[RequestInfo, ServedCandidatesForRequest]] = { + val predictionRequestId = query.features.flatMap(_.getOrElse(PredictionRequestIdFeature, None)) + val requestInfo = RequestInfo(query.getRequiredUserId, getProduct(query), predictionRequestId) + val selectedCandidatesInfo = selectedCandidates.map(getCandidateInfo) + val remainingCandidatesInfo = (remainingCandidates ++ droppedCandidates).map(getCandidateInfo) + Seq( + new ProducerRecord( + kafkaTopic, + requestInfo, + ServedCandidatesForRequest( + requestInfo, + selectedCandidatesInfo, + selectedCandidatesInfo ++ remainingCandidatesInfo + ) + ) + ) + } + + private def getProduct(query: PipelineQuery): Product = { + query.product match { + case HomeRecommendedTweetsProduct => Product.HomeRecommendedTweets + case VideoRecommendedTweetsProduct => Product.VideoRecommendedTweets + case _ => throw new IllegalArgumentException(s"Unsupported product type ${query.product}") + } + } + + private def getCandidateInfo(candidate: CandidateWithDetails): CandidateInfo = { + val sourceSignalInfo = candidate.features + .getOrElse(SignalInfoFeature, Seq.empty) + val sourceSignal = candidate.features.getTry(SourceSignalFeature).toOption + + CandidateInfo( + tweetId = candidate.candidateIdLong, + servedType = CandidateSourceUtil.getServedType(identifierPrefix, candidate.source.name), + score = candidate.features.getTry(ScoreFeature).toOption, + sourceSignalId = sourceSignal, + sourceSignalTypeInfo = Some( + sourceSignalInfo.map { s => + SignalInfo(signalType = s.signalType, timestamp = s.sourceEventTime.map(_.inMillis)) + } + ) + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/SelectedStatsSideEffect.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/SelectedStatsSideEffect.scala new file mode 100644 index 000000000..32be7b414 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect/SelectedStatsSideEffect.scala @@ -0,0 +1,142 @@ +package com.twitter.tweet_mixer.functional_component.side_effect + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.product_mixer.component_library.selector.Bucketer +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.HasVideo +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.IsLongVideo +import com.twitter.tweet_mixer.feature.TweetInfoFeatures.isFeatureSet +import com.twitter.tweet_mixer.feature.AuthorIdFeature +import com.twitter.tweet_mixer.feature.HydraScoreFeature +import com.twitter.tweet_mixer.feature.TweetBooleanInfoFeature +import com.twitter.tweet_mixer.feature.HighQualitySourceTweet +import com.twitter.tweet_mixer.feature.HighQualitySourceUser +import com.twitter.tweet_mixer.feature.SourceSignalFeature +import com.twitter.tweet_mixer.functional_component.hydrator.SGSFollowedUsersFeature +import com.twitter.tweet_mixer.functional_component.transformer.ReplyFeature +import com.twitter.tweet_mixer.model.response.TweetMixerResponse +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ExperimentBucketIdentifierParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.BlendingParam +import javax.inject.Inject +import javax.inject.Singleton +import com.twitter.tweet_mixer.feature.USSFeatures + +/** + * Side effect that calls hydra dark traffic to score tweets. + */ +@Singleton +class SelectedStatsSideEffect @Inject() ( + statsReceiver: StatsReceiver) + extends PipelineResultSideEffect[PipelineQuery, TweetMixerResponse] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("SelectedStats") + + val scopedStats = statsReceiver.scope("SelectedStats") + + def isVideo(candidateWithDetails: CandidateWithDetails): Boolean = { + isFeatureSet( + HasVideo, + candidateWithDetails.features.getOrElse(TweetBooleanInfoFeature, None).getOrElse(0) + ) + } + + def isLongVideo(candidateWithDetails: CandidateWithDetails): Boolean = { + isFeatureSet( + IsLongVideo, + candidateWithDetails.features.getOrElse(TweetBooleanInfoFeature, None).getOrElse(0) + ) + } + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, TweetMixerResponse] + ): Stitch[Unit] = { + val query = inputs.query + val allCandidates = (inputs.selectedCandidates ++ inputs.droppedCandidates) + .groupBy(_.candidateIdLong) + .map { case (candidateId, candidateSeq) => candidateSeq.head } + .toSeq + val selectedCandidates = inputs.selectedCandidates + + val videoSelectedCandidates = selectedCandidates.filter(isVideo) + val longVideoSelectedCandidates = + selectedCandidates.filter(candidate => isLongVideo(candidate) && isVideo(candidate)) + val replySelectedCandidates = selectedCandidates.filter { candidate => + candidate.features.getOrElse(ReplyFeature, false) + } + + val authorIds = selectedCandidates.map(_.features.getOrElse(AuthorIdFeature, None)) + val followedUserIds = + query.features.map(_.getOrElse(SGSFollowedUsersFeature, Nil)).getOrElse(Nil).toSet + val inNetworkAuthors = authorIds.filter(_.exists(followedUserIds.contains(_))) + + val unscoredCandidates = allCandidates.filter { candidate => + candidate.features.getOrElse(HydraScoreFeature, Map.empty[String, Double]).isEmpty + } + + // Fetch high quality source signals + val highQualitySourceTweetSignals = + USSFeatures.getSignals[Long](query, Set(HighQualitySourceTweet)) + val highQualitySourceUserSignals = + USSFeatures.getSignals[Long](query, Set(HighQualitySourceUser)) + + val selectedSourceSignals = selectedCandidates.map { candidate => + candidate.features.getOrElse(SourceSignalFeature, 0L) + } + + // Group candidates by pipeline with their source signals + val candidatesByPipeline = selectedCandidates + .zip(selectedSourceSignals) + .groupBy { case (candidate, _) => Bucketer.ByCandidateSource.apply(candidate) } + + val hasHighQualitySignals = + highQualitySourceTweetSignals.nonEmpty || highQualitySourceUserSignals.nonEmpty + + val scope = query.params(ExperimentBucketIdentifierParam) + + val blendingStrategy = query.params(BlendingParam) + + Stitch.value { + scopedStats.scope(scope).counter("AllCandidates").incr(allCandidates.size) + scopedStats.scope(scope).counter("SelectedCandidates").incr(selectedCandidates.size) + scopedStats + .scope(scope).counter("SelectedVideoCandidates").incr(videoSelectedCandidates.size) + scopedStats + .scope(scope).counter("SelectedLongVideoCandidates").incr(longVideoSelectedCandidates.size) + scopedStats.scope(scope).counter("SelectedReplyCandidates").incr(replySelectedCandidates.size) + scopedStats + .scope(scope).counter("SelectedInNetworkCandidates").incr(inNetworkAuthors.size) + scopedStats + .scope(scope).counter("UnscoredCandidates").incr(unscoredCandidates.size) + + if (hasHighQualitySignals) { + candidatesByPipeline.foreach { + case (pipelineId, candidatesWithSignals) => + val pipelineScope = + scopedStats.scope(scope).scope(blendingStrategy.toString).scope(pipelineId.name) + val signals = candidatesWithSignals.map(_._2) + pipelineScope + .counter("HighQualitySignalsPresentSelectedCandidates") + .incr(candidatesWithSignals.size) + // Track number of candidates belonging to each signal type + pipelineScope + .counter("SelectedHighQualitySourceTweetCandidates") + .incr(signals.count(highQualitySourceTweetSignals.contains)) + pipelineScope + .counter("SelectedHighQualitySourceUserCandidates") + .incr(signals.count(highQualitySourceUserSignals.contains)) + // Track unique number of signals used to retrieve candidates + pipelineScope + .counter("SelectedHighQualitySourceTweetUniqueSignals") + .incr(signals.filter(highQualitySourceTweetSignals.contains).toSet.size) + pipelineScope + .counter("SelectedHighQualitySourceUserUniqueSignals") + .incr(signals.filter(highQualitySourceUserSignals.contains).toSet.size) + } + } + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/AnnCandidateFeatureTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/AnnCandidateFeatureTransformer.scala new file mode 100644 index 000000000..7f954c58a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/AnnCandidateFeatureTransformer.scala @@ -0,0 +1,31 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.ann.common.CosineDistance +import com.twitter.ann.common.NeighborWithDistanceWithSeed +import com.twitter.tweet_mixer.feature.FromInNetworkSourceFeature +import com.twitter.tweet_mixer.feature.ScoreFeature +import com.twitter.tweet_mixer.feature.SourceSignalFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier + +object AnnCandidateFeatureTransformer + extends CandidateFeatureTransformer[NeighborWithDistanceWithSeed[Long, Long, CosineDistance]] { + override def features: Set[Feature[_, _]] = + Set(ScoreFeature, SourceSignalFeature, FromInNetworkSourceFeature) + + override val identifier: TransformerIdentifier = TransformerIdentifier("AnnCandidate") + + override def transform( + input: NeighborWithDistanceWithSeed[Long, Long, CosineDistance] + ): FeatureMap = + FeatureMap( + SourceSignalFeature, + input.seed, + ScoreFeature, + input.distance.distance.toDouble, + FromInNetworkSourceFeature, + false + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/BUILD.bazel new file mode 100644 index 000000000..87bf84300 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/BUILD.bazel @@ -0,0 +1,45 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "ann/src/main/scala/com/twitter/ann/common", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "simclusters-ann/thrift/src/main/thrift:thrift-scala", + "snowflake/src/main/scala/com/twitter/snowflake/id", + "src/java/com/twitter/common/text/language:locale-util", + "src/java/com/twitter/search/common/schema/base", + "src/java/com/twitter/search/common/schema/earlybird", + "src/java/com/twitter/search/queryparser", + "src/java/com/twitter/search/queryparser/query:core-query-nodes", + "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", + "src/scala/com/twitter/simclusters_v2/common", + "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", + "src/thrift/com/twitter/recos/user_tweet_graph:user_tweet_graph-scala", + "src/thrift/com/twitter/recos/user_video_graph:user_video_graph-scala", + "src/thrift/com/twitter/search:earlybird-scala", + "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", + "src/thrift/com/twitter/topic_recos:topic_recos-thrift-scala", + "timelineservice/common:model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request", + ], + exports = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/CertoTopicTweetsQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/CertoTopicTweetsQueryTransformer.scala new file mode 100644 index 000000000..9a83022df --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/CertoTopicTweetsQueryTransformer.scala @@ -0,0 +1,56 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.common.text.language.LocaleUtil +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.simclusters_v2.thriftscala.TopicId +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.candidate_source.topic_tweets.CertoTopicTweetsQuery +import com.twitter.tweet_mixer.feature.UserTopicIdsFeature + +case class CertoTopicTweetsQueryTransformer( + maxTweetsPerTopicParam: FSBoundedParam[Int], + maxTweetsParam: FSBoundedParam[Int], + minCertoScoreParam: FSBoundedParam[Double], + minCertoFavCountParam: FSBoundedParam[Int], + userInferredTopicIdsEnabled: FSParam[Boolean], + signalsFn: PipelineQuery => Seq[Long]) + extends CandidatePipelineQueryTransformer[PipelineQuery, CertoTopicTweetsQuery] { + + override def transform(inputQuery: PipelineQuery): CertoTopicTweetsQuery = { + val normalizedLanguage: Option[String] = + inputQuery.getLanguageCode.map { languageCode => + LocaleUtil.getLocaleOf(languageCode).getLanguage.toLowerCase + } + + val productContextTopicIds = signalsFn(inputQuery) + val topics: Seq[TopicId] = + if (productContextTopicIds.isEmpty && inputQuery.params(userInferredTopicIdsEnabled)) { + getUserSelectedTopics(inputQuery, language = normalizedLanguage, countryCode = None) + } else { + productContextTopicIds.map { topicId: Long => + TopicId(entityId = topicId, language = normalizedLanguage, country = None) + } + } + + CertoTopicTweetsQuery( + topicIds = topics, + maxCandidatesPerTopic = inputQuery.params(maxTweetsPerTopicParam), + maxCandidates = inputQuery.params(maxTweetsParam), + minCertoScore = inputQuery.params(minCertoScoreParam), + minFavCount = inputQuery.params(minCertoFavCountParam) + ) + } + + private def getUserSelectedTopics( + inputQuery: PipelineQuery, + language: Option[String], + countryCode: Option[String] + ): Seq[TopicId] = { + val topicIds = inputQuery.features.map(_.get(UserTopicIdsFeature)).getOrElse(Seq.empty) + topicIds.map { topicId: Long => + TopicId(entityId = topicId, language = language, country = countryCode) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EarlybirdInNetworkQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EarlybirdInNetworkQueryTransformer.scala new file mode 100644 index 000000000..768f9bc39 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EarlybirdInNetworkQueryTransformer.scala @@ -0,0 +1,130 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.conversions.DurationOps._ +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.HasExcludedIds +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.search.common.query.thriftjava.thriftscala.CollectorParams +import com.twitter.search.common.query.thriftjava.thriftscala.CollectorTerminationParams +import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.search.queryparser.query.{Query => SearchQuery} +import com.twitter.search.queryparser.query.search.SearchOperator +import com.twitter.timelines.util.SnowflakeSortIndexHelper +import com.twitter.util.Time +import com.twitter.search.queryparser.query.Conjunction +import com.twitter.tweet_mixer.candidate_source.earlybird_realtime_cg.InNetworkRequest +import com.twitter.tweet_mixer.functional_component.hydrator.HaploMissFeature +import com.twitter.tweet_mixer.functional_component.hydrator.SGSFollowedUsersFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MaxTweetAgeHoursParam +import scala.collection.JavaConverters._ + +object EarlybirdInNetworkQueryTransformer { + private val MaxFollowUsers = 1500 + private val NumEarlybirdPartitions = 3 + private val DefaultEarlybirdCandidates = 4500 + private val MaxEarlybirdCandidates = 6600 + + private val EarlybirdRequestMetadataOptions: eb.ThriftSearchResultMetadataOptions = + eb.ThriftSearchResultMetadataOptions( + getInReplyToStatusId = true, + getFromUserId = true, + getReferencedTweetAuthorId = true + ) +} + +case class EarlybirdInNetworkQueryTransformer[Query <: PipelineQuery with HasExcludedIds]( + candidatePipelineIdentifier: CandidatePipelineIdentifier, + clientId: Option[String]) + extends CandidatePipelineQueryTransformer[Query, InNetworkRequest] { + + import EarlybirdInNetworkQueryTransformer._ + + def createConjunction(clauses: Seq[SearchQuery]): Option[SearchQuery] = { + clauses.size match { + case 0 => None + case 1 => Some(clauses.head) + case _ => Some(new Conjunction(clauses.asJava)) + } + } + + def createRangeQuery( + beforeTweetIdExclusive: Option[Long], + afterTweetIdExclusive: Option[Long] + ): Option[SearchQuery] = { + val beforeIdClause = beforeTweetIdExclusive.map { beforeId => + new SearchOperator(SearchOperator.Type.SINCE_ID, beforeId.toString) + } + val afterIdClause = afterTweetIdExclusive.map { afterId => + // MAX_ID is an inclusive value therefore we subtract 1 from afterId + new SearchOperator(SearchOperator.Type.MAX_ID, (afterId - 1).toString) + } + createConjunction(Seq(beforeIdClause, afterIdClause).flatten) + } + + override def transform(query: Query): InNetworkRequest = { + + val duration = query.params(MaxTweetAgeHoursParam) + val sinceTime: Time = duration.ago + val untilTime: Time = Time.now + + val fromTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(sinceTime) + val toTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(untilTime) + + val followedUserIds = + query.features + .map(v => + v.getOrElse(SGSFollowedUsersFeature, Seq.empty[Long]) ++ Seq(query.getRequiredUserId)) + .map { _.take(MaxFollowUsers) } + + val excludedIds = query.excludedIds + + //Getting default number of candidates if there are no excludedIds. If there are excludedIds, get more tweets but cap it + val numResultsPerPartition = math.min( + DefaultEarlybirdCandidates + excludedIds.size, + MaxEarlybirdCandidates) / NumEarlybirdPartitions + + val collectorParams = CollectorParams( + numResultsToReturn = numResultsPerPartition, + terminationParams = Some( + CollectorTerminationParams( + maxHitsToProcess = Some(numResultsPerPartition), // return all Hits + timeoutMs = 200.milliseconds.inMilliseconds.toInt + )) + ) + + val rangeQuery = createRangeQuery(Some(fromTweetIdExclusive), Some(toTweetIdExclusive)) + val searchQuery: Option[SearchQuery] = rangeQuery + + val thriftQuery = eb.ThriftSearchQuery( + serializedQuery = searchQuery.map { _.serialize }, + fromUserIDFilter64 = followedUserIds, + maxHitsPerUser = -1, // disable maxHitsPerUser so we can avoid antigaming filter in Earlybird + numResults = -1, // disable + collectConversationId = true, + rankingMode = eb.ThriftSearchRankingMode.Recency, + relevanceOptions = None, + resultMetadataOptions = Some(EarlybirdRequestMetadataOptions), + collectorParams = Some(collectorParams), + searcherId = Some(query.getRequiredUserId), + searchStatusIds = None, + namedDisjunctionMap = None + ) + + val ebRequest = eb.EarlybirdRequest( + searchQuery = thriftQuery, + clientId = clientId, + getOlderResults = Some(false), + followedUserIds = followedUserIds, + getProtectedTweetsOnly = Some(false), + timeoutMs = 200.milliseconds.inMilliseconds.toInt, + skipVeryRecentTweets = true, + numResultsToReturnAtRoot = Some(numResultsPerPartition * NumEarlybirdPartitions) + ) + + val writeBackToHaplo = + query.features.getOrElse(FeatureMap.empty).getOrElse(HaploMissFeature, false) + InNetworkRequest(ebRequest, excludedIds, writeBackToHaplo) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EarlybirdInNetworkResponseFeatureTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EarlybirdInNetworkResponseFeatureTransformer.scala new file mode 100644 index 000000000..eb1c583ca --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EarlybirdInNetworkResponseFeatureTransformer.scala @@ -0,0 +1,52 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.tweet_mixer.feature.AuthorIdFeature +import com.twitter.tweet_mixer.feature.FromInNetworkSourceFeature +import com.twitter.tweet_mixer.feature.InReplyToTweetIdFeature +import com.twitter.tweet_mixer.feature.SourceTweetIdFeature +import com.twitter.search.earlybird.{thriftscala => eb} + +object ReplyFeature extends Feature[TweetCandidate, Boolean] + +object EarlybirdInNetworkResponseFeatureTransformer + extends CandidateFeatureTransformer[eb.ThriftSearchResult] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("EarlybirdInNetworkResponse") + + override def features: Set[Feature[_, _]] = + Set( + AuthorIdFeature, + SourceTweetIdFeature, + FromInNetworkSourceFeature, + ReplyFeature, + InReplyToTweetIdFeature) + + override def transform(input: eb.ThriftSearchResult): FeatureMap = { + val isRetweet = input.metadata.flatMap(_.isRetweet) + val isReply = input.metadata.flatMap(_.isReply) + val inReplyToTweetId = + if (isReply.getOrElse(false)) input.metadata.map(_.sharedStatusId) else None + val sourceTweetId = + if (isRetweet.getOrElse(false)) + input.metadata.map(_.sharedStatusId) + else Some(input.id) + val authorId = + if (isRetweet.getOrElse(false)) + input.metadata.map(_.referencedTweetAuthorId) + else input.metadata.map(_.fromUserId) + FeatureMapBuilder(sizeHint = 5) + .add(AuthorIdFeature, authorId) + .add(ReplyFeature, isReply.getOrElse(false)) + .add(SourceTweetIdFeature, sourceTweetId) + .add(InReplyToTweetIdFeature, inReplyToTweetId) + .add(FromInNetworkSourceFeature, true) + .build() + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EvergreenVideosQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EvergreenVideosQueryTransformer.scala new file mode 100644 index 000000000..78a426dfd --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EvergreenVideosQueryTransformer.scala @@ -0,0 +1,33 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.EvergreenVideosSearchByUserIdsQuery +import com.twitter.tweet_mixer.functional_component.hydrator.SGSFollowedUsersFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenVideosMaxFollowUsersParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.EvergreenVideosPaginationNumParam + +case class EvergreenVideosQueryTransformer[Query <: PipelineQuery]( + candidatePipelineIdentifier: CandidatePipelineIdentifier) + extends CandidatePipelineQueryTransformer[Query, EvergreenVideosSearchByUserIdsQuery] { + + override def transform(query: Query): EvergreenVideosSearchByUserIdsQuery = { + + val paginationNum = query.params(EvergreenVideosPaginationNumParam) + val maxFollowUsers = query.params(EvergreenVideosMaxFollowUsersParam) + + val followedUserIds = + query.features + .map(v => + v.getOrElse(SGSFollowedUsersFeature, Seq.empty[Long]) ++ Seq(query.getRequiredUserId)) + .map { + _.take(maxFollowUsers) + } + + EvergreenVideosSearchByUserIdsQuery( + userIds = followedUserIds.getOrElse(Seq.empty), + size = paginationNum + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EvergreenVideosResponseFeatureTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EvergreenVideosResponseFeatureTransformer.scala new file mode 100644 index 000000000..6a48a5763 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/EvergreenVideosResponseFeatureTransformer.scala @@ -0,0 +1,30 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.tweet_mixer.feature.FromInNetworkSourceFeature +import com.twitter.tweet_mixer.feature.ScoreFeature +import com.twitter.tweet_mixer.feature.SourceSignalFeature +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate + +object EvergreenVideosResponseFeatureTransformer + extends CandidateFeatureTransformer[TweetMixerCandidate] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("EvergreenVideos") + + override def features: Set[Feature[_, _]] = + Set(SourceSignalFeature, FromInNetworkSourceFeature, ScoreFeature) + + override def transform(input: TweetMixerCandidate): FeatureMap = + FeatureMap( + SourceSignalFeature, + input.seedId, + FromInNetworkSourceFeature, + false, + ScoreFeature, + input.score + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/GrokTopicTweetsQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/GrokTopicTweetsQueryTransformer.scala new file mode 100644 index 000000000..ff740c370 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/GrokTopicTweetsQueryTransformer.scala @@ -0,0 +1,31 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.common.text.language.LocaleUtil +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.tweet_mixer.candidate_source.popular_grok_topic_tweets.GrokTopicTweetsQuery +import com.twitter.tweet_mixer.feature.RequestCountryPlaceIdFeature + +case class GrokTopicTweetsQueryTransformer(maxNumCandidatesParam: FSBoundedParam[Int]) + extends CandidatePipelineQueryTransformer[PipelineQuery, GrokTopicTweetsQuery] { + + override def transform(inputQuery: PipelineQuery): GrokTopicTweetsQuery = { + val normalizedLanguage: Option[String] = + inputQuery.getLanguageCode.map { languageCode => + LocaleUtil.getLocaleOf(languageCode).getLanguage.toLowerCase + } + + val countryPlaceId = getCountryPlaceId(inputQuery) + GrokTopicTweetsQuery( + inputQuery.getOptionalUserId.getOrElse(0L), + normalizedLanguage, + countryPlaceId, + inputQuery.params(maxNumCandidatesParam) + ) + } + + private def getCountryPlaceId(inputQuery: PipelineQuery): Option[Long] = { + inputQuery.features.map(_.get(RequestCountryPlaceIdFeature)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/HaploliteResponseFeatureTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/HaploliteResponseFeatureTransformer.scala new file mode 100644 index 000000000..00f1c8def --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/HaploliteResponseFeatureTransformer.scala @@ -0,0 +1,45 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.timelineservice.model.Tweet +import com.twitter.tweet_mixer.feature.AuthorIdFeature +import com.twitter.tweet_mixer.feature.FromInNetworkSourceFeature +import com.twitter.tweet_mixer.feature.InReplyToTweetIdFeature +import com.twitter.tweet_mixer.feature.SourceTweetIdFeature +object HaploliteResponseFeatureTransformer extends CandidateFeatureTransformer[Tweet] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("HaploliteResponse") + + override def features: Set[Feature[_, _]] = + Set( + AuthorIdFeature, + SourceTweetIdFeature, + FromInNetworkSourceFeature, + ReplyFeature, + InReplyToTweetIdFeature) + + override def transform(input: Tweet): FeatureMap = { + val isRetweet = input.isRetweet + val isReply = input.isReply + val sourceTweetId = + if (isRetweet) + input.sourceTweetId + else Some(input.tweetId) + val authorId = + if (isRetweet) + input.sourceUserId + else input.userId + FeatureMapBuilder(sizeHint = 5) + .add(AuthorIdFeature, authorId) + .add(ReplyFeature, isReply) + .add(SourceTweetIdFeature, sourceTweetId) + .add(FromInNetworkSourceFeature, true) + .add(InReplyToTweetIdFeature, input.inReplyToTweetId) + .build() + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/QigBatchQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/QigBatchQueryTransformer.scala new file mode 100644 index 000000000..aa3f71e69 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/QigBatchQueryTransformer.scala @@ -0,0 +1,34 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.search.query_interaction_graph.service.{thriftscala => t} +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.trends.common.thriftscala.UserIdentifier +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.QigSearchHistoryTopKLimit + +case class QigBatchQueryTransformer[Query <: PipelineQuery]( + signalFn: PipelineQuery => Seq[String], +) extends CandidatePipelineQueryTransformer[ + Query, + Seq[t.QigRequest] + ] { + + override def transform(inputQuery: Query): Seq[t.QigRequest] = { + val queries: Seq[String] = signalFn(inputQuery) + + queries.map { query => + t.QigRequest( + query = Some(query), + userIdentifier = Some( + UserIdentifier( + userId = inputQuery.getOptionalUserId, + guestIdMarketing = inputQuery.clientContext.guestIdMarketing + ) + ), + userInfo = Some(t.UserInfo(inputQuery.clientContext.userRoles)), + productContext = Some(t.ProductContext.TweetContext(t.TweetContext())), + topKLimit = Some(inputQuery.params(QigSearchHistoryTopKLimit)), + ) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/QigTweetCandidateFeatureTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/QigTweetCandidateFeatureTransformer.scala new file mode 100644 index 000000000..6f41e5812 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/QigTweetCandidateFeatureTransformer.scala @@ -0,0 +1,27 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.tweet_mixer.candidate_source.qig_service.QigTweetCandidate +import com.twitter.tweet_mixer.feature.LanguageCodeFeature +import com.twitter.tweet_mixer.feature.SearcherRealtimeHistorySourceSignalFeature + +object QigQueryFeature extends Feature[PipelineQuery, String] + +object QigTweetCandidateFeatureTransformer extends CandidateFeatureTransformer[QigTweetCandidate] { + override val identifier: TransformerIdentifier = TransformerIdentifier("QigTweetCandidate") + override def features: Set[Feature[_, _]] = + Set(QigQueryFeature, LanguageCodeFeature, SearcherRealtimeHistorySourceSignalFeature) + + override def transform(input: QigTweetCandidate): FeatureMap = { + FeatureMapBuilder() + .add(QigQueryFeature, input.query) + .add(LanguageCodeFeature, input.tweetCandidate.qigTweetFeatures.flatMap(_.language)) + .add(SearcherRealtimeHistorySourceSignalFeature, input.query) + .build() + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/SANNQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/SANNQueryTransformer.scala new file mode 100644 index 000000000..c74a852df --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/SANNQueryTransformer.scala @@ -0,0 +1,61 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.simclusters_v2.common.ModelVersions +import com.twitter.simclusters_v2.thriftscala.EmbeddingType +import com.twitter.simclusters_v2.thriftscala.InternalId +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.candidate_source.simclusters_ann.SANNQuery +import com.twitter.tweet_mixer.config.SimClustersANNConfig +import com.twitter.tweet_mixer.feature.USSFeatures +import com.twitter.tweet_mixer.param.SimClustersANNParams.EnableSANNCacheParam +import com.twitter.tweet_mixer.param.SimClustersANNParams.ModelVersionParam + +case class SANNQueryTransformer( + override val identifier: TransformerIdentifier, + clusterParamMap: Map[_ <: FSParam[Boolean], _ <: FSParam[String]], + signalsFn: PipelineQuery => Seq[InternalId], + embeddingTypes: Seq[EmbeddingType], + minScoreParam: FSBoundedParam[Double], + maxInterestedInCandidatesParam: Option[FSBoundedParam[Int]] = None) + extends CandidatePipelineQueryTransformer[PipelineQuery, SANNQuery] { + + /* This determines the total number of candidates from Simclusters ANN + */ + val maxSANNCandidates: Int = 10000 + + override def transform(inputQuery: PipelineQuery): SANNQuery = { + val signalLength = + USSFeatures + .getSignals(inputQuery, USSFeatures.TweetFeatures ++ USSFeatures.ProducerFeatures) + .length + val numCandidates = maxInterestedInCandidatesParam match { + case Some(param) => inputQuery.params(param) + case None => if (signalLength == 0) 200 else maxSANNCandidates / signalLength + } + val configIds: Seq[String] = clusterParamMap.collect { + case (enableParam, configParam) if inputQuery.params(enableParam) => + inputQuery.params(configParam) + }.toSeq + val modelVersion = { + ModelVersions.Enum.enumToSimClustersModelVersionMap(inputQuery.params(ModelVersionParam)) + } + val minScore = inputQuery.params(minScoreParam) + val queries = signalsFn(inputQuery).flatMap { signal => + configIds.flatMap { configId => + embeddingTypes.map { embeddingType => + SimClustersANNConfig.getQuery( + signal, + embeddingType, + modelVersion, + configId + ) + } + } + } + SANNQuery(queries, numCandidates, minScore, inputQuery.params(EnableSANNCacheParam)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/SkitTopicTweetsQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/SkitTopicTweetsQueryTransformer.scala new file mode 100644 index 000000000..0f4389107 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/SkitTopicTweetsQueryTransformer.scala @@ -0,0 +1,83 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.conversions.DurationOps._ +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.contentrecommender.thriftscala.AlgorithmType +import com.twitter.simclusters_v2.thriftscala.EmbeddingType +import com.twitter.timelines.configapi.FSParam +import com.twitter.simclusters_v2.thriftscala.ModelVersion +import com.twitter.topic_recos.thriftscala.TopicTweetPartitionFlatKey +import com.twitter.tweet_mixer.candidate_source.topic_tweets.SkitTimedTopicKeys +import com.twitter.tweet_mixer.candidate_source.topic_tweets.SkitTopicTweetsQuery +import com.twitter.tweet_mixer.feature.UserTopicIdsFeature +import com.twitter.tweet_mixer.functional_component.transformer.SkitTopicTweetsQueryTransformer._ +import com.twitter.util.Duration + +case class SkitTopicTweetsQueryTransformer( + maxTweetsPerTopicParam: FSBoundedParam[Int], + maxTweetsParam: FSBoundedParam[Int], + minSkitScoreParam: FSBoundedParam[Double], + minSkitFavCountParam: FSBoundedParam[Int], + userInferredTopicIdsEnabled: FSParam[Boolean], + maxTweetAgeInHours: FSBoundedParam[Duration], + semanticCoreVersionIdParam: FSBoundedParam[Long], + algorithmType: Option[AlgorithmType], + simclustersModelVersion: Option[ModelVersion], + signalsFn: PipelineQuery => Seq[Long]) + extends CandidatePipelineQueryTransformer[PipelineQuery, SkitTopicTweetsQuery] { + + assert(algorithmType.exists(algorithm => AllowedAlgorithmTypes.contains(algorithm))) + + override def transform(inputQuery: PipelineQuery): SkitTopicTweetsQuery = { + + val productContextTopicIds = signalsFn(inputQuery) + + val topics: Seq[Long] = + if (productContextTopicIds.isEmpty && inputQuery.params(userInferredTopicIdsEnabled)) { + inputQuery.features.map(_.get(UserTopicIdsFeature)).getOrElse(Seq.empty) + } else { + productContextTopicIds + } + + val latestTweetTimeInHour = inputQuery.queryTime.inHours.toLong + + val earliestTweetTimeInHour = latestTweetTimeInHour - + math.min(MaxTweetAgeInHours, inputQuery.params(maxTweetAgeInHours).inHours).toLong + + val skitTimedTopicKeys = topics.map { topicId => + val timedTopicKeys = + for (timePartition <- earliestTweetTimeInHour to latestTweetTimeInHour) yield { + + TopicTweetPartitionFlatKey( + entityId = topicId, + timePartition = timePartition, + algorithmType = algorithmType, + tweetEmbeddingType = Some(EmbeddingType.LogFavBasedTweet), + language = inputQuery.getLanguageCode.getOrElse("").toLowerCase, + country = None, // Disable country. It is not used. + semanticCoreAnnotationVersionId = Some(inputQuery.params(semanticCoreVersionIdParam)), + simclustersModelVersion = simclustersModelVersion, + ) + } + + SkitTimedTopicKeys(keys = timedTopicKeys, topicId = topicId) + } + + SkitTopicTweetsQuery( + topicKeys = skitTimedTopicKeys, + maxCandidatesPerTopic = inputQuery.params(maxTweetsPerTopicParam), + maxCandidates = inputQuery.params(maxTweetsParam), + minScore = inputQuery.params(minSkitScoreParam), + minFavCount = inputQuery.params(minSkitFavCountParam) + ) + } +} + +object SkitTopicTweetsQueryTransformer { + val MaxTweetAgeInHours: Int = 7.days.inHours // Simple guard to prevent overloading + + val AllowedAlgorithmTypes: Set[AlgorithmType] = + Set(AlgorithmType.TfgTweet, AlgorithmType.SemanticCoreTweet) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TimelineQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TimelineQueryTransformer.scala new file mode 100644 index 000000000..4217e99cc --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TimelineQueryTransformer.scala @@ -0,0 +1,51 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelineservice.{thriftscala => tlsthrift} +import com.twitter.tweet_mixer.feature.AuthorIdFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MediaRelatedCreatorMaxCandidates + +case class TimelineQueryTransformer( + timelineType: tlsthrift.TimelineType, + signalFn: PipelineQuery => Seq[Long], + disableUrtHealthTreatments: Option[Boolean]) + extends CandidatePipelineQueryTransformer[ + PipelineQuery, + tlsthrift.TimelineQuery + ] { + + override def transform( + query: PipelineQuery + ): tlsthrift.TimelineQuery = { + val timelineQueryOptions: tlsthrift.TimelineQueryOptions = tlsthrift.TimelineQueryOptions( + contextualUserId = query.getOptionalUserId, + deviceContext = Some(toDeviceContext(query.clientContext)), + contextualUserContext = + Some(tlsthrift.ContextualUserContext(roles = query.clientContext.userRoles)), + disableUrtHealthTreatments = disableUrtHealthTreatments + ) + + val authorId = query.features.get.getOrElse(AuthorIdFeature, None).getOrElse(-1L) + + tlsthrift.TimelineQuery( + timelineType = timelineType, + authorId, + maxCount = query.params(MediaRelatedCreatorMaxCandidates).toShort, + options = Some(timelineQueryOptions) + ) + } + + private def toDeviceContext(clientContext: ClientContext): tlsthrift.DeviceContext = { + tlsthrift.DeviceContext( + countryCode = clientContext.countryCode, + languageCode = clientContext.languageCode, + clientAppId = clientContext.appId, + ipAddress = clientContext.ipAddress, + guestId = clientContext.guestId, + deviceId = clientContext.deviceId, + userAgent = clientContext.userAgent + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TopicTweetFeatureTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TopicTweetFeatureTransformer.scala new file mode 100644 index 000000000..17312cf0a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TopicTweetFeatureTransformer.scala @@ -0,0 +1,23 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.tweet_mixer.feature.TopicTweetScore +import com.twitter.tweet_mixer.feature.TweetTopicIdFeature +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate + +object TopicTweetFeatureTransformer extends CandidateFeatureTransformer[TweetMixerCandidate] { + override def features: Set[Feature[_, _]] = Set(TopicTweetScore, TweetTopicIdFeature) + + override val identifier: TransformerIdentifier = TransformerIdentifier( + "TopicTweetFeatureTransformer") + + override def transform(candidate: TweetMixerCandidate): FeatureMap = + FeatureMapBuilder() + .add(TopicTweetScore, Some(candidate.score)) + .add(TweetTopicIdFeature, candidate.seedId) + .build() +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripStratoGeoQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripStratoGeoQueryTransformer.scala new file mode 100644 index 000000000..561d1ebc1 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripStratoGeoQueryTransformer.scala @@ -0,0 +1,56 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import com.twitter.tweet_mixer.candidate_source.popular_geo_tweets.TripStratoGeoQuery +import com.twitter.tweet_mixer.feature.RequestCountryPlaceIdFeature +import com.twitter.common.text.language.LocaleUtil + +import com.twitter.logging.Logger + +case class TripStratoGeoQueryTransformer( + geoSourceIdsParam: FSParam[Seq[String]], + maxTweetsPerDomainParam: FSBoundedParam[Int], + maxPopGeoTweetsParam: FSBoundedParam[Int]) + extends CandidatePipelineQueryTransformer[PipelineQuery, TripStratoGeoQuery] { + private val log = Logger.get(classOf[TripStratoGeoQueryTransformer]) + + override def transform(inputQuery: PipelineQuery): TripStratoGeoQuery = { + val normalizedLanguage: Option[String] = + inputQuery.getLanguageCode.map { languageCode => + LocaleUtil.getLocaleOf(languageCode).getLanguage.toLowerCase + } + val domains = getDomains( + inputQuery = inputQuery, + languages = Seq(normalizedLanguage, None) + ) + + TripStratoGeoQuery( + domains = domains, + maxCandidatesPerSource = inputQuery.params(maxTweetsPerDomainParam), + maxPopGeoCandidates = inputQuery.params(maxPopGeoTweetsParam) + ) + } + + private def getDomains( + inputQuery: PipelineQuery, + languages: Seq[Option[String]] + ): Seq[TripDomain] = { + val candidateSourceIds: Seq[String] = inputQuery.params(geoSourceIdsParam) + + val countryPlaceId: Option[Long] = getCountryPlaceId(inputQuery) + + candidateSourceIds.flatMap { candidateSourceId => + languages.map { language => + TripDomain(sourceId = candidateSourceId, language = language, placeId = countryPlaceId) + } + } + } + + private def getCountryPlaceId(inputQuery: PipelineQuery): Option[Long] = { + inputQuery.features.map(_.get(RequestCountryPlaceIdFeature)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripStratoTopicQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripStratoTopicQueryTransformer.scala new file mode 100644 index 000000000..6a270ceaf --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripStratoTopicQueryTransformer.scala @@ -0,0 +1,76 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import com.twitter.tweet_mixer.candidate_source.popular_topic_tweets.TripStratoTopicQuery +import com.twitter.tweet_mixer.feature.RequestCountryPlaceIdFeature +import com.twitter.tweet_mixer.feature.UserTopicIdsFeature +import com.twitter.logging.Logger +import com.twitter.common.text.language.LocaleUtil + +case class TripStratoTopicQueryTransformer( + sourceIdsParam: FSParam[Seq[String]], + maxTweetsPerDomainParam: FSBoundedParam[Int], + maxTweetsParam: FSBoundedParam[Int], + popTopicIdsParam: FSParam[Seq[Long]]) + extends CandidatePipelineQueryTransformer[PipelineQuery, TripStratoTopicQuery] { + private val log = Logger.get(classOf[TripStratoTopicQueryTransformer]) + + override def transform(inputQuery: PipelineQuery): TripStratoTopicQuery = { + val normalizedLanguage: Option[String] = + inputQuery.getLanguageCode.map { languageCode => + LocaleUtil.getLocaleOf(languageCode).getLanguage.toLowerCase + } + val domains = + getDomains(inputQuery = inputQuery, languages = Seq(normalizedLanguage, None).distinct) + + TripStratoTopicQuery( + domains = domains, + maxCandidatesPerSource = inputQuery.params(maxTweetsPerDomainParam), + maxPopTopicCandidates = inputQuery.params(maxTweetsParam) + ) + } + + private def getDomains( + inputQuery: PipelineQuery, + languages: Seq[Option[String]] + ): Seq[TripDomain] = { + val candidateSourceIds: Seq[String] = inputQuery.params(sourceIdsParam) + + val countryPlaceId: Option[Long] = getCountryPlaceId(inputQuery) + val userSelectedTopicIds: Seq[Long] = getUserSelectedTopicIds(inputQuery) + val topicIds: Seq[Long] = { + // only use popular topic ids if user did not select any topics + if (userSelectedTopicIds.isEmpty) { + val popTopicIds: Seq[Long] = inputQuery.params(popTopicIdsParam) + popTopicIds + } else { + userSelectedTopicIds + } + } + + candidateSourceIds.flatMap { candidateSourceId => + languages.map { language => + topicIds.map { topicId => + TripDomain( + sourceId = candidateSourceId, + language = language, + placeId = countryPlaceId, + topicId = Some(topicId) + ) + } + } + }.flatten + } + + private def getCountryPlaceId(inputQuery: PipelineQuery): Option[Long] = { + inputQuery.features.map(_.get(RequestCountryPlaceIdFeature)) + } + + private def getUserSelectedTopicIds(inputQuery: PipelineQuery): Seq[Long] = { + inputQuery.features.map(_.getOrElse(UserTopicIdsFeature, Seq.empty)).getOrElse(Seq.empty) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripTweetFeatureTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripTweetFeatureTransformer.scala new file mode 100644 index 000000000..c1e915376 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TripTweetFeatureTransformer.scala @@ -0,0 +1,19 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.tweet_mixer.feature.TripTweetScore +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet + +object TripTweetFeatureTransformer extends CandidateFeatureTransformer[TripTweet] { + override def features: Set[Feature[_, _]] = Set(TripTweetScore) + + override val identifier: TransformerIdentifier = TransformerIdentifier( + "TripTweetCandidateFeature") + + override def transform(input: TripTweet): FeatureMap = + FeatureMapBuilder().add(TripTweetScore, Some(input.score)).build() +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TweetFeatureTimelineServiceTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TweetFeatureTimelineServiceTransformer.scala new file mode 100644 index 000000000..b8077a4fe --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TweetFeatureTimelineServiceTransformer.scala @@ -0,0 +1,30 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.timelineservice.{thriftscala => tlsthrift} +import com.twitter.tweet_mixer.feature.FromInNetworkSourceFeature +import com.twitter.tweet_mixer.feature.ScoreFeature +import com.twitter.tweet_mixer.feature.SourceSignalFeature + +object TweetFeatureTimelineServiceTransformer extends CandidateFeatureTransformer[tlsthrift.Tweet] { + override def features: Set[Feature[_, _]] = + Set(ScoreFeature, SourceSignalFeature, FromInNetworkSourceFeature) + + override val identifier: TransformerIdentifier = TransformerIdentifier( + "TweetFeatureTimelineService") + + override def transform( + input: tlsthrift.Tweet + ): FeatureMap = + FeatureMap( + SourceSignalFeature, + input._1, + ScoreFeature, + 1.0, + FromInNetworkSourceFeature, + false + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TweetMixerCandidateFeatureTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TweetMixerCandidateFeatureTransformer.scala new file mode 100644 index 000000000..d753e23e6 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TweetMixerCandidateFeatureTransformer.scala @@ -0,0 +1,30 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.tweet_mixer.feature.FromInNetworkSourceFeature +import com.twitter.tweet_mixer.feature.ScoreFeature +import com.twitter.tweet_mixer.feature.SourceSignalFeature +import com.twitter.tweet_mixer.model.response.TweetMixerCandidate + +object TweetMixerCandidateFeatureTransformer + extends CandidateFeatureTransformer[TweetMixerCandidate] { + override def features: Set[Feature[_, _]] = + Set(ScoreFeature, SourceSignalFeature, FromInNetworkSourceFeature) + + override val identifier: TransformerIdentifier = TransformerIdentifier("SANNCandidateFeature") + + override def transform( + input: TweetMixerCandidate + ): FeatureMap = + FeatureMap( + SourceSignalFeature, + input.seedId, + ScoreFeature, + input.score, + FromInNetworkSourceFeature, + false + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TwitterClipV0LongVideoQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TwitterClipV0LongVideoQueryTransformer.scala new file mode 100644 index 000000000..21bfcdf04 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TwitterClipV0LongVideoQueryTransformer.scala @@ -0,0 +1,32 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.EvergreenVideosSearchByTweetQuery +import com.twitter.tweet_mixer.functional_component.hydrator.SeedsTextFeatures +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LongFormMaxDurationParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LongFormMaxResultParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LongFormMinDurationParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LongFormMinHeightParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LongFormMinWidthParam + +case class TwitterClipV0LongVideoQueryTransformer[Query <: PipelineQuery]( + signalFn: PipelineQuery => Seq[Long], + candidatePipelineIdentifier: CandidatePipelineIdentifier) + extends CandidatePipelineQueryTransformer[Query, EvergreenVideosSearchByTweetQuery] { + + override def transform(query: Query): EvergreenVideosSearchByTweetQuery = { + val textMap = query.features.get.getOrElse(SeedsTextFeatures, None) + + EvergreenVideosSearchByTweetQuery( + tweetIds = signalFn(query), + textMap = textMap, + size = query.params(LongFormMaxResultParam), + minWidth = query.params(LongFormMinWidthParam), + minHeight = query.params(LongFormMinHeightParam), + minDurationSec = query.params(LongFormMinDurationParam).inSeconds, + maxDurationSec = query.params(LongFormMaxDurationParam).inSeconds + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TwitterClipV0ShortVideoQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TwitterClipV0ShortVideoQueryTransformer.scala new file mode 100644 index 000000000..ef02a58e3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/TwitterClipV0ShortVideoQueryTransformer.scala @@ -0,0 +1,32 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.tweet_mixer.candidate_source.evergreen_videos.EvergreenVideosSearchByTweetQuery +import com.twitter.tweet_mixer.functional_component.hydrator.SeedsTextFeatures +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ShortFormMaxDurationParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ShortFormMaxResultParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ShortFormMinDurationParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ShortFormMinHeightParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ShortFormMinWidthParam + +case class TwitterClipV0ShortVideoQueryTransformer[Query <: PipelineQuery]( + signalFn: PipelineQuery => Seq[Long], + candidatePipelineIdentifier: CandidatePipelineIdentifier) + extends CandidatePipelineQueryTransformer[Query, EvergreenVideosSearchByTweetQuery] { + + override def transform(query: Query): EvergreenVideosSearchByTweetQuery = { + val textMap = query.features.get.getOrElse(SeedsTextFeatures, None) + + EvergreenVideosSearchByTweetQuery( + tweetIds = signalFn(query), + textMap = textMap, + size = query.params(ShortFormMaxResultParam), + minWidth = query.params(ShortFormMinWidthParam), + minHeight = query.params(ShortFormMinHeightParam), + minDurationSec = query.params(ShortFormMinDurationParam).inSeconds, + maxDurationSec = query.params(ShortFormMaxDurationParam).inSeconds + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UTGProducerBasedQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UTGProducerBasedQueryTransformer.scala new file mode 100644 index 000000000..ba6f5d65a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UTGProducerBasedQueryTransformer.scala @@ -0,0 +1,38 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.tweet_mixer.candidate_source.UTG.UTGProducerBasedRequest +import com.twitter.tweet_mixer.feature.EntityTypes.UserId +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MaxCandidateNumPerSourceKeyParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MaxTweetAgeHoursParam +import com.twitter.tweet_mixer.param.UTGParams.EnableUTGCacheParam +import com.twitter.tweet_mixer.param.UTGParams.MaxNumFollowersParam +import com.twitter.tweet_mixer.param.UTGParams.MinCoOccurrenceParam +import com.twitter.tweet_mixer.param.UTGParams.SimilarityAlgorithm +import com.twitter.tweet_mixer.param.UTGParams.SimilarityAlgorithmEnum + +case class UTGProducerBasedQueryTransformer( + override val identifier: TransformerIdentifier, + signalsFn: PipelineQuery => Seq[UserId], + minScoreParam: FSBoundedParam[Double]) + extends CandidatePipelineQueryTransformer[PipelineQuery, UTGProducerBasedRequest] { + + override def transform(inputQuery: PipelineQuery): UTGProducerBasedRequest = { + val params = inputQuery.params + val tweetSignals = signalsFn(inputQuery) + UTGProducerBasedRequest( + tweetSignals, + maxResults = Some(params(MaxCandidateNumPerSourceKeyParam)), + minCooccurrence = Some(params(MinCoOccurrenceParam)), + minScore = Some(params(minScoreParam)), + maxTweetAgeInHours = Some(params(MaxTweetAgeHoursParam).inHours), + maxNumFollowers = Some(params(MaxNumFollowersParam)), + similarityAlgorithm = + Some(SimilarityAlgorithmEnum.enumToSimilarityAlgorithmMap(params(SimilarityAlgorithm))), + enableCache = inputQuery.params(EnableUTGCacheParam) + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UTGTweetBasedQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UTGTweetBasedQueryTransformer.scala new file mode 100644 index 000000000..e804df74b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UTGTweetBasedQueryTransformer.scala @@ -0,0 +1,67 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.tweet_mixer.candidate_source.UTG.UTGTweetBasedRequest +import com.twitter.tweet_mixer.feature.EntityTypes.TweetId +import com.twitter.tweet_mixer.functional_component.hydrator.UTGOutlierSignalsFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MaxCandidateNumPerSourceKeyParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MaxTweetAgeHoursParam +import com.twitter.tweet_mixer.param.UTGParams.CoverageExpansionOldTweetEnabledParam +import com.twitter.tweet_mixer.param.UTGParams.EnableUTGCacheParam +import com.twitter.tweet_mixer.param.UTGParams.MaxConsumerSeedsNumParam +import com.twitter.tweet_mixer.param.UTGParams.MinCoOccurrenceParam +import com.twitter.tweet_mixer.param.UTGParams.SimilarityAlgorithm +import com.twitter.tweet_mixer.param.UTGParams.SimilarityAlgorithmEnum +import com.twitter.util.Duration +import com.twitter.util.Time +import scala.concurrent.duration.HOURS + +case class UTGTweetBasedQueryTransformer( + override val identifier: TransformerIdentifier, + signalsFn: PipelineQuery => Seq[TweetId], + isExpansionQuery: Boolean, + minScoreParam: FSBoundedParam[Double], + degreeExponent: FSBoundedParam[Double]) + extends CandidatePipelineQueryTransformer[PipelineQuery, UTGTweetBasedRequest] { + + private val oldTweetThreshold: Duration = Duration(48, HOURS) + + override def transform(inputQuery: PipelineQuery): UTGTweetBasedRequest = { + val params = inputQuery.params + val tweetSignals = signalsFn(inputQuery) + val expansionModeEnabled = params(CoverageExpansionOldTweetEnabledParam) + val tweetSignalsPostExpansion = { + if (expansionModeEnabled && isExpansionQuery) tweetSignals.filter(isOldTweet) + else if (expansionModeEnabled && !isExpansionQuery) tweetSignals.filterNot(isOldTweet) + else if (!isExpansionQuery) tweetSignals + else Nil + } + + val outliers = inputQuery.features.get.getOrElse(UTGOutlierSignalsFeature, Set.empty[Long]) + + val filteredPosts = tweetSignalsPostExpansion.filter { id => + !outliers.contains(id) + } + + UTGTweetBasedRequest( + filteredPosts, + maxResults = Some(params(MaxCandidateNumPerSourceKeyParam)), + minCooccurrence = Some(params(MinCoOccurrenceParam)), + minScore = Some(params(minScoreParam)), + maxTweetAgeInHours = Some(params(MaxTweetAgeHoursParam).inHours), + maxConsumerSeeds = Some(params(MaxConsumerSeedsNumParam)), + similarityAlgorithm = + Some(SimilarityAlgorithmEnum.enumToSimilarityAlgorithmMap(params(SimilarityAlgorithm))), + enableCache = inputQuery.params(EnableUTGCacheParam), + degreeExponent = Some(params(degreeExponent)) + ) + } + + private def isOldTweet(tweetId: TweetId): Boolean = { + SnowflakeId.timeFromIdOpt(tweetId).exists(_ < Time.now - oldTweetThreshold) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UVGTweetBasedQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UVGTweetBasedQueryTransformer.scala new file mode 100644 index 000000000..6a57eca95 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UVGTweetBasedQueryTransformer.scala @@ -0,0 +1,77 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.tweet_mixer.candidate_source.UVG.UVGTweetBasedRequest +import com.twitter.tweet_mixer.feature.EntityTypes.TweetId +import com.twitter.tweet_mixer.functional_component.hydrator.UVGOutlierSignalsFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MaxCandidateNumPerSourceKeyParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MaxTweetAgeHoursParam +import com.twitter.tweet_mixer.param.UVGParams.CoverageExpansionOldTweetEnabledParam +import com.twitter.tweet_mixer.param.UVGParams.EnableUVGCacheParam +import com.twitter.tweet_mixer.param.UVGParams.MaxConsumerSeedsNumParam +import com.twitter.tweet_mixer.param.UVGParams.MaxLeftNodeDegreeParam +import com.twitter.tweet_mixer.param.UVGParams.MaxNumSamplesPerNeighborParam +import com.twitter.tweet_mixer.param.UVGParams.MaxRightNodeDegreeParam +import com.twitter.tweet_mixer.param.UVGParams.MinCoOccurrenceParam +import com.twitter.tweet_mixer.param.UVGParams.SampleRHSTweetsParam +import com.twitter.tweet_mixer.param.UVGParams.SimilarityAlgorithm +import com.twitter.tweet_mixer.param.UVGParams.SimilarityAlgorithmEnum +import com.twitter.util.Duration +import com.twitter.util.Time + +import scala.concurrent.duration.HOURS + +case class UVGTweetBasedQueryTransformer( + override val identifier: TransformerIdentifier, + signalsFn: PipelineQuery => Seq[TweetId], + isExpansionQuery: Boolean, + minScoreParam: FSBoundedParam[Double], + degreeExponent: FSBoundedParam[Double]) + extends CandidatePipelineQueryTransformer[PipelineQuery, UVGTweetBasedRequest] { + + private val oldTweetThreshold: Duration = Duration(48, HOURS) + + override def transform(inputQuery: PipelineQuery): UVGTweetBasedRequest = { + val params = inputQuery.params + val tweetSignals = signalsFn(inputQuery) + + val expansionModeEnabled = params(CoverageExpansionOldTweetEnabledParam) + val tweetSignalsPostExpansion = { + if (expansionModeEnabled && isExpansionQuery) tweetSignals.filter(isOldTweet) + else if (expansionModeEnabled && !isExpansionQuery) tweetSignals.filterNot(isOldTweet) + else if (!isExpansionQuery) tweetSignals + else Nil + } + + val outliers = inputQuery.features.get.getOrElse(UVGOutlierSignalsFeature, Set.empty[Long]) + + val filteredPosts = tweetSignalsPostExpansion.filter { id => + !outliers.contains(id) + } + + UVGTweetBasedRequest( + filteredPosts, + maxResults = Some(params(MaxCandidateNumPerSourceKeyParam)), + minCooccurrence = Some(params(MinCoOccurrenceParam)), + minScore = Some(params(minScoreParam)), + maxTweetAgeInHours = Some(params(MaxTweetAgeHoursParam).inHours), + maxConsumerSeeds = Some(params(MaxConsumerSeedsNumParam)), + similarityAlgorithm = + Some(SimilarityAlgorithmEnum.enumToSimilarityAlgorithmMap(params(SimilarityAlgorithm))), + enableCache = inputQuery.params(EnableUVGCacheParam), + maxNumSamplesPerNeighbor = Some(inputQuery.params(MaxNumSamplesPerNeighborParam)), + maxLeftNodeDegree = Some(inputQuery.params(MaxLeftNodeDegreeParam)), + maxRightNodeDegree = Some(inputQuery.params(MaxRightNodeDegreeParam)), + sampleRHSTweets = Some(inputQuery.params(SampleRHSTweetsParam)), + degreeExponent = Some(params(degreeExponent)) + ) + } + + private def isOldTweet(tweetId: TweetId): Boolean = { + SnowflakeId.timeFromIdOpt(tweetId).exists(_ < Time.now - oldTweetThreshold) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UtegQueryTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UtegQueryTransformer.scala new file mode 100644 index 000000000..a5cb32ba4 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UtegQueryTransformer.scala @@ -0,0 +1,57 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.tweet_mixer.feature.RealGraphInNetworkScoresFeature +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.HasExcludedIds +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.recos.recos_common.thriftscala.SocialProofType +import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendationType +import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetEntityDisplayLocation +import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.MaxTweetAgeHoursParam +import com.twitter.util.Time + +object UtegQueryTransformer { + private val MaxUserSocialProofSize = 10 + private val MaxTweetSocialProofSize = 10 + private val MinUserSocialProofSize = 1 + private val MaxTweetsToFetch = 800 + private val MaxExcludedTweets = 1500 +} + +case class UtegQueryTransformer[Query <: PipelineQuery with HasExcludedIds]( + candidatePipelineIdentifier: CandidatePipelineIdentifier) + extends CandidatePipelineQueryTransformer[Query, uteg.RecommendTweetEntityRequest] { + + import UtegQueryTransformer._ + + override def transform(query: Query): uteg.RecommendTweetEntityRequest = { + val weightedFollowings = query.features + .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double])) + .getOrElse(Map.empty) + + val duration = query.params(MaxTweetAgeHoursParam) + + val sinceTime: Time = duration.ago + + val excludedTweetIds = query.excludedIds + + uteg.RecommendTweetEntityRequest( + requesterId = query.getRequiredUserId, + displayLocation = TweetEntityDisplayLocation.HomeTimeline, + recommendationTypes = Seq(RecommendationType.Tweet), + seedsWithWeights = weightedFollowings, + maxResultsByType = Some(Map(RecommendationType.Tweet -> MaxTweetsToFetch)), + maxTweetAgeInMillis = Some(sinceTime.untilNow.inMillis), + excludedTweetIds = Some(excludedTweetIds.toSeq), + maxUserSocialProofSize = Some(MaxUserSocialProofSize), + maxTweetSocialProofSize = Some(MaxTweetSocialProofSize), + minUserSocialProofSizes = Some(Map(RecommendationType.Tweet -> MinUserSocialProofSize)), + socialProofTypes = Some(Seq(SocialProofType.Favorite)), + tweetAuthors = None, + maxEngagementAgeInMillis = None, + tweetTypes = None + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UtegResponseFeatureTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UtegResponseFeatureTransformer.scala new file mode 100644 index 000000000..b2485e63a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer/UtegResponseFeatureTransformer.scala @@ -0,0 +1,30 @@ +package com.twitter.tweet_mixer.functional_component.transformer + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} +import com.twitter.tweet_mixer.feature.FromInNetworkSourceFeature +import com.twitter.tweet_mixer.feature.ScoreFeature +import com.twitter.tweet_mixer.feature.SourceSignalFeature + +object UtegResponseFeatureTransformer + extends CandidateFeatureTransformer[uteg.TweetRecommendation] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("UtegResponse") + + override def features: Set[Feature[_, _]] = + Set(ScoreFeature, SourceSignalFeature, FromInNetworkSourceFeature) + + override def transform(input: uteg.TweetRecommendation): FeatureMap = + FeatureMap( + ScoreFeature, + input.score, + SourceSignalFeature, + -1L, + FromInNetworkSourceFeature, + false + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/BUILD.bazel new file mode 100644 index 000000000..849d514af --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/BUILD.bazel @@ -0,0 +1,27 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/debugger_tweets/marshaller/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/imv_recommended_tweets/marshaller/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/imv_related_tweets/marshaller/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/logged_out_video_recommended_tweets/marshaller/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/notifications_recommended_tweets/marshaller/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/rux_related_tweets/marshaller/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/topic_tweets/marshaller/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/video_recommended_tweets/marshaller/request", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], + exports = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerDebugParamsUnmarshaller.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerDebugParamsUnmarshaller.scala new file mode 100644 index 000000000..7af2d24e8 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerDebugParamsUnmarshaller.scala @@ -0,0 +1,28 @@ +package com.twitter.tweet_mixer.marshaller.request + +import com.twitter.product_mixer.core.functional_component.marshaller.request.FeatureValueUnmarshaller +import com.twitter.product_mixer.core.model.marshalling.request.DebugParams +import com.twitter.tweet_mixer.model.request.TweetMixerDebugOptions +import com.twitter.tweet_mixer.{thriftscala => t} +import com.twitter.util.Time +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TweetMixerDebugParamsUnmarshaller @Inject() ( + featureValueUnmarshaller: FeatureValueUnmarshaller) { + + def apply(debugParams: t.DebugParams): DebugParams = { + DebugParams( + featureOverrides = debugParams.featureOverrides.map { map => + map.mapValues(featureValueUnmarshaller(_)).toMap + }, + debugOptions = debugParams.debugOptions.map { options => + TweetMixerDebugOptions( + requestTimeOverride = options.requestTimeOverrideMillis.map(Time.fromMilliseconds), + showIntermediateLogs = options.showIntermediateLogs.orElse(Some(false)) + ) + } + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerProductContextUnmarshaller.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerProductContextUnmarshaller.scala new file mode 100644 index 000000000..aa0352b24 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerProductContextUnmarshaller.scala @@ -0,0 +1,52 @@ +package com.twitter.tweet_mixer.marshaller.request + +import com.twitter.product_mixer.core.model.marshalling.request.ProductContext +import com.twitter.tweet_mixer.product.home_recommended_tweets.marshaller.request.HomeRecommendedTweetsProductContextUnmarshaller +import com.twitter.tweet_mixer.product.notifications_recommended_tweets.marshaller.request.NotificationsRecommendedTweetsProductContextUnmarshaller +import com.twitter.tweet_mixer.product.imv_recommended_tweets.marshaller.request.IMVRecommendedTweetsProductContextUnmarshaller +import com.twitter.tweet_mixer.product.imv_related_tweets.marshaller.request.IMVRelatedTweetsProductContextUnmarshaller +import com.twitter.tweet_mixer.product.rux_related_tweets.marshaller.request.RUXRelatedTweetsProductContextUnmarshaller +import com.twitter.tweet_mixer.product.debugger_tweets.marshaller.request.DebuggerTweetsProductContextUnmarshaller +import com.twitter.tweet_mixer.product.video_recommended_tweets.marshaller.request.VideoRecommendedTweetsProductContextUnmarshaller +import com.twitter.tweet_mixer.product.topic_tweets.marshaller.request.TopicTweetsProductContextUnmarshaller +import com.twitter.tweet_mixer.product.logged_out_video_recommended_tweets.marshaller.request.LoggedOutVideoRecommendedTweetsProductContextUnmarshaller +import com.twitter.tweet_mixer.{thriftscala => t} + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TweetMixerProductContextUnmarshaller @Inject() ( + homeRecommendedTweetsProductContextUnmarshaller: HomeRecommendedTweetsProductContextUnmarshaller, + notificationsRecommendedTweetsProductContextUnmarshaller: NotificationsRecommendedTweetsProductContextUnmarshaller, + imvRecommendedTweetsProductContextUnmarshaller: IMVRecommendedTweetsProductContextUnmarshaller, + imvRelatedTweetsProductContextUnmarshaller: IMVRelatedTweetsProductContextUnmarshaller, + ruxRelatedTweetsProductContextUnmarshaller: RUXRelatedTweetsProductContextUnmarshaller, + videoRecommendedTweetsProductContextUnmarshaller: VideoRecommendedTweetsProductContextUnmarshaller, + loggedOutVideoRecommendedTweetsProductContextUnmarshaller: LoggedOutVideoRecommendedTweetsProductContextUnmarshaller, + topicTweetsProductContextUnmarshaller: TopicTweetsProductContextUnmarshaller, + debuggerTweetsProductContextUnmarshaller: DebuggerTweetsProductContextUnmarshaller) { + + def apply(productContext: t.ProductContext): ProductContext = productContext match { + case t.ProductContext.HomeRecommendedTweetsProductContext(context) => + homeRecommendedTweetsProductContextUnmarshaller(context) + case t.ProductContext.NotificationsRecommendedTweetsProductContext(context) => + notificationsRecommendedTweetsProductContextUnmarshaller(context) + case t.ProductContext.ImvRecommendedTweetsProductContext(context) => + imvRecommendedTweetsProductContextUnmarshaller(context) + case t.ProductContext.ImvRelatedTweetsProductContext(context) => + imvRelatedTweetsProductContextUnmarshaller(context) + case t.ProductContext.RuxRelatedTweetsProductContext(context) => + ruxRelatedTweetsProductContextUnmarshaller(context) + case t.ProductContext.DebuggerTweetsProductContext(context) => + debuggerTweetsProductContextUnmarshaller(context) + case t.ProductContext.VideoRecommendedTweetsProductContext(context) => + videoRecommendedTweetsProductContextUnmarshaller(context) + case t.ProductContext.LoggedOutVideoRecommendedTweetsProductContext(context) => + loggedOutVideoRecommendedTweetsProductContextUnmarshaller(context) + case t.ProductContext.TopicTweetsProductContext(context) => + topicTweetsProductContextUnmarshaller(context) + case t.ProductContext.UnknownUnionField(field) => + throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}") + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerProductUnmarshaller.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerProductUnmarshaller.scala new file mode 100644 index 000000000..8b1cdabe4 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerProductUnmarshaller.scala @@ -0,0 +1,34 @@ +package com.twitter.tweet_mixer.marshaller.request + +import com.twitter.product_mixer.core.model.marshalling.request.Product +import com.twitter.tweet_mixer.model.request.DebuggerTweetsProduct +import com.twitter.tweet_mixer.model.request.HomeRecommendedTweetsProduct +import com.twitter.tweet_mixer.model.request.IMVRecommendedTweetsProduct +import com.twitter.tweet_mixer.model.request.IMVRelatedTweetsProduct +import com.twitter.tweet_mixer.model.request.LoggedOutVideoRecommendedTweetsProduct +import com.twitter.tweet_mixer.model.request.NotificationsRecommendedTweetsProduct +import com.twitter.tweet_mixer.model.request.RUXRelatedTweetsProduct +import com.twitter.tweet_mixer.model.request.TopicTweetsProduct +import com.twitter.tweet_mixer.model.request.VideoRecommendedTweetsProduct +import com.twitter.tweet_mixer.{thriftscala => t} + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TweetMixerProductUnmarshaller @Inject() () { + + def apply(product: t.Product): Product = product match { + case t.Product.HomeRecommendedTweets => HomeRecommendedTweetsProduct + case t.Product.NotificationsRecommendedTweets => NotificationsRecommendedTweetsProduct + case t.Product.ImvRecommendedTweets => IMVRecommendedTweetsProduct + case t.Product.ImvRelatedTweets => IMVRelatedTweetsProduct + case t.Product.RuxRelatedTweets => RUXRelatedTweetsProduct + case t.Product.VideoRecommendedTweets => VideoRecommendedTweetsProduct + case t.Product.TopicTweets => TopicTweetsProduct + case t.Product.LoggedOutVideoRecommendedTweets => LoggedOutVideoRecommendedTweetsProduct + case t.Product.DebuggerTweets => DebuggerTweetsProduct + case t.Product.EnumUnknownProduct(value) => + throw new UnsupportedOperationException(s"Unknown product: $value") + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerRequestUnmarshaller.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerRequestUnmarshaller.scala new file mode 100644 index 000000000..38a4d65f2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/request/TweetMixerRequestUnmarshaller.scala @@ -0,0 +1,30 @@ +package com.twitter.tweet_mixer.marshaller.request + +import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextUnmarshaller +import com.twitter.tweet_mixer.model.request.TweetMixerRequest +import com.twitter.tweet_mixer.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TweetMixerRequestUnmarshaller @Inject() ( + clientContextUnmarshaller: ClientContextUnmarshaller, + tweetMixerProductUnmarshaller: TweetMixerProductUnmarshaller, + tweetMixerProductContextUnmarshaller: TweetMixerProductContextUnmarshaller, + tweetMixerDebugParamsUnmarshaller: TweetMixerDebugParamsUnmarshaller) { + + def apply(tweetMixerRequest: t.TweetMixerRequest): TweetMixerRequest = { + TweetMixerRequest( + clientContext = clientContextUnmarshaller(tweetMixerRequest.clientContext), + product = tweetMixerProductUnmarshaller(tweetMixerRequest.product), + productContext = + tweetMixerRequest.productContext.map(tweetMixerProductContextUnmarshaller(_)), + // Avoid de-serializing cursors in the request unmarshaller. The unmarshaller should never + // fail, which is often a possibility when trying to de-serialize a cursor. Cursors can also + // be product-specific and more appropriately handled in individual product pipelines. + serializedRequestCursor = tweetMixerRequest.cursor, + maxResults = tweetMixerRequest.maxResults, + debugParams = tweetMixerRequest.debugParams.map(tweetMixerDebugParamsUnmarshaller(_)), + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/BUILD.bazel new file mode 100644 index 000000000..ba452e861 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/BUILD.bazel @@ -0,0 +1,36 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/debugger_tweets/marshaller/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/debugger_tweets/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/imv_recommended_tweets/marshaller/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/imv_recommended_tweets/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/imv_related_tweets/marshaller/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/imv_related_tweets/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/logged_out_video_recommended_tweets/marshaller/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/logged_out_video_recommended_tweets/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/notifications_recommended_tweets/marshaller/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/notifications_recommended_tweets/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/rux_related_tweets/marshaller/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/rux_related_tweets/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/topic_tweets/marshaller/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/topic_tweets/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/video_recommended_tweets/marshaller/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/video_recommended_tweets/model/response", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], + exports = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/TweetMixerProductResponseMarshaller.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/TweetMixerProductResponseMarshaller.scala new file mode 100644 index 000000000..605d7a8c3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/TweetMixerProductResponseMarshaller.scala @@ -0,0 +1,79 @@ +package com.twitter.tweet_mixer.marshaller.response + +import com.twitter.tweet_mixer.model.response.TweetMixerProductResponse +import com.twitter.tweet_mixer.product.home_recommended_tweets.marshaller.response.HomeRecommendedTweetsProductResponseMarshaller +import com.twitter.tweet_mixer.product.home_recommended_tweets.model.response.HomeRecommendedTweetsProductResponse +import com.twitter.tweet_mixer.product.imv_recommended_tweets.marshaller.response.IMVRecommendedTweetsProductResponseMarshaller +import com.twitter.tweet_mixer.product.imv_recommended_tweets.model.response.IMVRecommendedTweetsProductResponse +import com.twitter.tweet_mixer.product.imv_related_tweets.marshaller.response.IMVRelatedTweetsProductResponseMarshaller +import com.twitter.tweet_mixer.product.imv_related_tweets.model.response.IMVRelatedTweetsProductResponse +import com.twitter.tweet_mixer.product.logged_out_video_recommended_tweets.marshaller.response.LoggedOutVideoRecommendedTweetsProductResponseMarshaller +import com.twitter.tweet_mixer.product.logged_out_video_recommended_tweets.model.response.LoggedOutVideoRecommendedTweetsProductResponse +import com.twitter.tweet_mixer.product.notifications_recommended_tweets.marshaller.response.NotificationsRecommendedTweetsProductResponseMarshaller +import com.twitter.tweet_mixer.product.notifications_recommended_tweets.model.response.NotificationsRecommendedTweetsProductResponse +import com.twitter.tweet_mixer.product.rux_related_tweets.marshaller.response.RUXRelatedTweetsProductResponseMarshaller +import com.twitter.tweet_mixer.product.rux_related_tweets.model.response.RUXRelatedTweetsProductResponse +import com.twitter.tweet_mixer.product.debugger_tweets.marshaller.response.DebuggerTweetsProductResponseMarshaller +import com.twitter.tweet_mixer.product.debugger_tweets.model.response.DebuggerTweetsProductResponse +import com.twitter.tweet_mixer.product.topic_tweets.marshaller.response.TopicTweetsProductResponseMarshaller +import com.twitter.tweet_mixer.product.topic_tweets.model.response.TopicsTweetsProductResponse +import com.twitter.tweet_mixer.product.video_recommended_tweets.marshaller.response.VideoRecommendedTweetsProductResponseMarshaller +import com.twitter.tweet_mixer.product.video_recommended_tweets.model.response.VideoRecommendedTweetsProductResponse +import com.twitter.tweet_mixer.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +/** + * When adding a new Product, you will need to marshal the domain Product Response into Thrift. + * + * @see [[HomeRecommendedTweetsProductResponseMarshaller]] for an example. + */ +@Singleton +class TweetMixerProductResponseMarshaller @Inject() ( + homeRecommendedTweetsProductResponseMarshaller: HomeRecommendedTweetsProductResponseMarshaller, + notificationsRecommendedTweetsProductResponseMarshaller: NotificationsRecommendedTweetsProductResponseMarshaller, + imvRecommendedTweetsProductResponseMarshaller: IMVRecommendedTweetsProductResponseMarshaller, + imvRelatedTweetsProductResponseMarshaller: IMVRelatedTweetsProductResponseMarshaller, + ruxRelatedTweetsProductResponseMarshaller: RUXRelatedTweetsProductResponseMarshaller, + videoRecommendedTweetsProductResponseMarshaller: VideoRecommendedTweetsProductResponseMarshaller, + loggedOutVideoRecommendedTweetsProductResponseMarshaller: LoggedOutVideoRecommendedTweetsProductResponseMarshaller, + topicTweetsProductResponseMarshaller: TopicTweetsProductResponseMarshaller, + debuggerTweetsProductResponseMarshaller: DebuggerTweetsProductResponseMarshaller) { + + def apply( + tweetMixerProductResponse: TweetMixerProductResponse + ): t.TweetMixerRecommendationResponse = + tweetMixerProductResponse match { + case productResponse: HomeRecommendedTweetsProductResponse => + homeRecommendedTweetsProductResponseMarshaller(productResponse) + + case productResponse: NotificationsRecommendedTweetsProductResponse => + notificationsRecommendedTweetsProductResponseMarshaller(productResponse) + + case productResponse: IMVRecommendedTweetsProductResponse => + imvRecommendedTweetsProductResponseMarshaller(productResponse) + + case productResponse: IMVRelatedTweetsProductResponse => + imvRelatedTweetsProductResponseMarshaller(productResponse) + + case productResponse: RUXRelatedTweetsProductResponse => + ruxRelatedTweetsProductResponseMarshaller(productResponse) + + case productResponse: DebuggerTweetsProductResponse => + debuggerTweetsProductResponseMarshaller(productResponse) + + case productResponse: VideoRecommendedTweetsProductResponse => + videoRecommendedTweetsProductResponseMarshaller(productResponse) + + case productResponse: LoggedOutVideoRecommendedTweetsProductResponse => + loggedOutVideoRecommendedTweetsProductResponseMarshaller(productResponse) + + case productResponse: TopicsTweetsProductResponse => + topicTweetsProductResponseMarshaller(productResponse) + + case _ => + throw new UnsupportedOperationException( + s"Unknown product response: $tweetMixerProductResponse" + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/TweetMixerResponseTransportMarshaller.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/TweetMixerResponseTransportMarshaller.scala new file mode 100644 index 000000000..e8ce8da5a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/TweetMixerResponseTransportMarshaller.scala @@ -0,0 +1,26 @@ +package com.twitter.tweet_mixer.marshaller.response + +import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller +import com.twitter.product_mixer.core.model.common.identifier.TransportMarshallerIdentifier +import com.twitter.tweet_mixer.model.response.TweetMixerResponse +import com.twitter.tweet_mixer.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Marshals a domain response into a Thrift response. + * + * NOTE: You will most likely not need to modify this. + */ +@Singleton +class TweetMixerResponseTransportMarshaller @Inject() ( + tweetMixerProductResponseMarshaller: TweetMixerProductResponseMarshaller) + extends TransportMarshaller[TweetMixerResponse, t.TweetMixerRecommendationResponse] { + + val identifier: TransportMarshallerIdentifier = + TransportMarshallerIdentifier("TweetMixerResponse") + + def apply(tweetMixerResponse: TweetMixerResponse): t.TweetMixerRecommendationResponse = { + tweetMixerProductResponseMarshaller(tweetMixerResponse.tweetMixerProductResponse) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/common/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/common/BUILD.bazel new file mode 100644 index 000000000..ae7e978d2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/common/BUILD.bazel @@ -0,0 +1,17 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/javax/inject:javax.inject", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], + exports = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/common/TweetResultMarshaller.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/common/TweetResultMarshaller.scala new file mode 100644 index 000000000..8a9510c1f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/common/TweetResultMarshaller.scala @@ -0,0 +1,25 @@ +package com.twitter.tweet_mixer.marshaller.response.common + +import com.twitter.tweet_mixer.model.response.TweetResult +import com.twitter.tweet_mixer.{thriftscala => t} +import javax.inject.Singleton + +@Singleton +class TweetResultMarshaller { + def apply(tweetResult: TweetResult): t.TweetResult = t.TweetResult( + tweetId = tweetResult.id, + score = Some(tweetResult.score), + metricTags = Some(tweetResult.metricTags), + tweetMetadata = tweetResult.metadata.map { metadata => + t.TweetMetadata( + sourceSignalId = metadata.sourceSignalId, + signalType = metadata.signalType, + servedType = metadata.servedType, + signalEntity = metadata.signalEntity, + authorId = metadata.authorId, + ) + }, + inReplyToTweetId = tweetResult.inReplyToTweetId, + authorId = tweetResult.authorId + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/BUILD.bazel new file mode 100644 index 000000000..54e356a77 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/BUILD.bazel @@ -0,0 +1,8 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/ModuleNames.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/ModuleNames.scala new file mode 100644 index 000000000..84d7046e6 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/ModuleNames.scala @@ -0,0 +1,58 @@ +package com.twitter.tweet_mixer.model + +object ModuleNames { + // ANN Service clients + final val TwHINRegularUpdateAnnServiceClientName = "TwHINRegularUpdateAnnServiceClient" + + // ANN MH Embedding Providers + final val TwHINEmbeddingRegularUpdateMhEmbeddingProducer = + "TwHINEmbeddingRegularUpdateMhEmbeddingProducer" + final val ConsumerBasedTwHINEmbeddingRegularUpdateMhEmbeddingProducer = + "ConsumerBasedTwHINEmbeddingRegularUpdateMhEmbeddingProducer" + + // ANN queryable by id + final val TwHINAnnQueryableById = "TwHINAnnQueryableById" + final val ConsumerBasedTwHINAnnQueryableById = "ConsumerBasedTwHINAnnQueryableById" + + // SANN clients + final val ProdSimClustersANNServiceClientName = "ProdSimClustersANNServiceClient" + final val SimClustersVideoANNServiceClientName = "SimClustersVideoANNServiceClient" + + // In-memory Caches + final val TweeypieInMemCache = "TweeypieInMemCache" + final val MediaMetadataInMemCache = "MediaMetadataInMemCache" + final val GrokFilterInMemCache = "GrokFilterInMemCache" + + // Manhattan Client + final val ManhattanAthenaClient = "ManhattanAthenaClient" + final val ManhattanApolloClient = "ManhattanApolloClient" + + // Manhattan Repo + final val RealGraphInNetworkScoresOnPremRepo = "RealGraphInNetworkScoresOnPremRepo" + + // Twhin ANN Client + final val TwHINANNServiceClient = "TwHINANNServiceClient" + final val TwHINTweetEmbeddingStratoStore = "TwHINTweetEmbeddingStratoStore" + final val TwhinRebuildUserPositiveEmbeddingsStore = "TwhinRebuildUserPositiveEmbeddingsStore" + final val VecDBAnnServiceClient = "VecDBAnnServiceClient" + final val GPURetrievalProdHttpClient = "GPURetrievalProdHttpClient" + final val GPURetrievalDevelHttpClient = "GPURetrievalDevelHttpClient" + + // TwHIN rebuild ANN + final val TwHINRebuildTweetEmbeddingStratoStore = "TwHINRebuildTweetEmbeddingStratoStore" + + // Certo Topic Tweets + final val CertoStratoTopicTweetsStoreName = "CertoStratoTopicTweetsStore" + + final val SkitStratoTopicTweetsStoreName = "SkitTopicTweetsStore" + + final val EarlybirdRealtimeCGEndpoint = "EarlybirdRealtimeCGEndpoint" + + final val MemcachedImpressionBloomFilterStore = "MemcachedImpressionBloomFilterStore" + + final val MemcachedImpressionVideoBloomFilterStore = "MemcachedImpressionVideoBloomFilterStore" + + // Event publishers + final val ServedCandidatesScribeEventPublisher = "ServedCandidatesScribeEventPublisher" + +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/BUILD.bazel new file mode 100644 index 000000000..f26f291c1 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/BUILD.bazel @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], + exports = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasContentCategory.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasContentCategory.scala new file mode 100644 index 000000000..1443f1c58 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasContentCategory.scala @@ -0,0 +1,5 @@ +package com.twitter.tweet_mixer.model.request + +trait HasContentCategory { + def contentCategoryIds: Option[Seq[Long]] +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasTopicIds.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasTopicIds.scala new file mode 100644 index 000000000..e15889fac --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasTopicIds.scala @@ -0,0 +1,5 @@ +package com.twitter.tweet_mixer.model.request + +trait HasTopicIds { + def topicIds: Option[Seq[Long]] +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasVideoType.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasVideoType.scala new file mode 100644 index 000000000..26b78d080 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/HasVideoType.scala @@ -0,0 +1,7 @@ +package com.twitter.tweet_mixer.model.request + +import com.twitter.tweet_mixer.{thriftscala => t} + +trait HasVideoType { + def videoType: Option[t.VideoType] +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerDebugOptions.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerDebugOptions.scala new file mode 100644 index 000000000..de7d3db5b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerDebugOptions.scala @@ -0,0 +1,9 @@ +package com.twitter.tweet_mixer.model.request + +import com.twitter.product_mixer.core.model.marshalling.request.DebugOptions +import com.twitter.util.Time + +case class TweetMixerDebugOptions( + override val requestTimeOverride: Option[Time], + override val showIntermediateLogs: Option[Boolean]) + extends DebugOptions diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerProduct.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerProduct.scala new file mode 100644 index 000000000..7abaeca95 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerProduct.scala @@ -0,0 +1,55 @@ +package com.twitter.tweet_mixer.model.request + +import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.Product + +/** + * Identifier names on products can be used to create Feature Switch rules by product, + * which useful if bucketing occurs in a component shared by multiple products. + * @see [[Product.identifier]] + */ + +case object HomeRecommendedTweetsProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("HomeRecommendedTweets") + override val stringCenterProject: Option[String] = Some("tweet-mixer-home-recommended-tweets") +} + +case object NotificationsRecommendedTweetsProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("NotificationsRecommendedTweets") + override val stringCenterProject: Option[String] = Some( + "tweet-mixer-notifications-recommended-tweets") +} +case object IMVRecommendedTweetsProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("IMVRecommendedTweets") + override val stringCenterProject: Option[String] = Some("tweet-mixer-imv-recommended-tweets") +} + +case object IMVRelatedTweetsProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("IMVRelatedTweets") + override val stringCenterProject: Option[String] = Some("tweet-mixer-imv-related-tweets") +} + +case object TopicTweetsProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("TopicTweets") + override val stringCenterProject: Option[String] = Some("tweet-mixer-topic-tweets") +} +case object RUXRelatedTweetsProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("RUXRelatedTweets") + override val stringCenterProject: Option[String] = Some("tweet-mixer-rux-related-tweets") +} + +case object VideoRecommendedTweetsProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("VideoRecommendedTweets") + override val stringCenterProject: Option[String] = Some("tweet-mixer-video-recommended-tweets") +} + +case object LoggedOutVideoRecommendedTweetsProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("LoggedOutVideoRecommendedTweets") + override val stringCenterProject: Option[String] = Some( + "tweet-mixer-logged-out-video-recommended-tweets") +} + +case object DebuggerTweetsProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("DebuggerTweets") + override val stringCenterProject: Option[String] = Some("tweet-mixer-debugger-tweets") +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerRequest.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerRequest.scala new file mode 100644 index 000000000..1bd8bbb48 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request/TweetMixerRequest.scala @@ -0,0 +1,17 @@ +package com.twitter.tweet_mixer.model.request + +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.product_mixer.core.model.marshalling.request.DebugParams +import com.twitter.product_mixer.core.model.marshalling.request.Product +import com.twitter.product_mixer.core.model.marshalling.request.ProductContext +import com.twitter.product_mixer.core.model.marshalling.request.Request + +case class TweetMixerRequest( + override val clientContext: ClientContext, + override val product: Product, + // Product-specific parameters should be placed in the Product Context + override val productContext: Option[ProductContext], + override val serializedRequestCursor: Option[String], + override val maxResults: Option[Int], + override val debugParams: Option[DebugParams]) + extends Request diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/BUILD.bazel new file mode 100644 index 000000000..65bc8c14d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/BUILD.bazel @@ -0,0 +1,11 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/RecommendationResult.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/RecommendationResult.scala new file mode 100644 index 000000000..de4fc0776 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/RecommendationResult.scala @@ -0,0 +1,15 @@ +package com.twitter.tweet_mixer.model.response + +import com.twitter.product_mixer.core.model.marshalling.HasMarshalling +import com.twitter.tweet_mixer.{thriftscala => t} + +sealed trait RecommendationResult extends HasMarshalling + +case class TweetResult( + id: Long, + score: Double, + metricTags: Seq[t.MetricTag], + metadata: Option[t.TweetMetadata] = None, + inReplyToTweetId: Option[Long] = None, + authorId: Option[Long] = None) + extends RecommendationResult diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerCandidate.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerCandidate.scala new file mode 100644 index 000000000..96daed06c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerCandidate.scala @@ -0,0 +1,14 @@ +package com.twitter.tweet_mixer.model.response + +import com.twitter.tweet_mixer.utils.Utils + +case class TweetMixerCandidate( + tweetId: Long, + score: Double, + seedId: Long) + +object TweetMixerCandidate { + private val keyFn: TweetMixerCandidate => Long = candidate => candidate.tweetId + def interleave(candidates: Seq[Seq[TweetMixerCandidate]]): Seq[TweetMixerCandidate] = + Utils.interleave(candidates, keyFn) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerProductResponse.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerProductResponse.scala new file mode 100644 index 000000000..4e6ac873b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerProductResponse.scala @@ -0,0 +1,16 @@ +package com.twitter.tweet_mixer.model.response + +/** + * When adding a new Product, we need to define a domain model for the response. This will be + * marshaled into a Thrift, so these case classes should correspond 1:1 with their respective Thrift + * definitions. They should contain no business logic. + * + * This file should mostly correspond with the product_response.thrift file. The domain models for + * the shared response components should be added to the common directory. + * + * NOTE: You should not use these case classes for anything other than domain marshalling or + * marshalling to Thrift. + * + * @see [[com.twitter.tweet_mixer.product.home_recommended_tweets.model.response.HomeRecommendedTweetsProductResponse]] for an example. + */ +trait TweetMixerProductResponse diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerResponse.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerResponse.scala new file mode 100644 index 000000000..1069fbb0f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response/TweetMixerResponse.scala @@ -0,0 +1,15 @@ +package com.twitter.tweet_mixer.model.response + +import com.twitter.product_mixer.core.model.marshalling.HasMarshalling +import com.twitter.product_mixer.core.model.marshalling.HasLength + +case class TweetMixerResponse(tweetMixerProductResponse: TweetMixerProductResponse) + extends HasMarshalling + with HasLength { + override def length: Int = { + tweetMixerProductResponse match { + case response: HasLength => response.length + case _ => 1 + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/BUILD.bazel new file mode 100644 index 000000000..9b5af5e02 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/BUILD.bazel @@ -0,0 +1,40 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/com/twitter/bijection:scrooge", + "3rdparty/jvm/com/twitter/storehaus:memcache", + "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", + "haplolite-thrift/thrift/src/main/thrift:thrift-scala", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", + "hydra/embedding-generation/thrift/src/main/thrift/com/twitter/hydra/embedding_generation:thrift-scala", + "hydra/root/thrift/src/main/thrift:thrift-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", + "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client", + "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client", + "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection", + "servo/manhattan", + "simclusters-ann/thrift/src/main/thrift:thrift-scala", + "snowflake/src/main/scala/com/twitter/snowflake/id", + "src/scala/com/twitter/ml/featurestore/lib/params", + "src/scala/com/twitter/simclusters_v2/common", + "src/scala/com/twitter/storehaus_internal/manhattan", + "src/scala/com/twitter/storehaus_internal/manhattan/config", + "src/scala/com/twitter/storehaus_internal/memcache", + "src/scala/com/twitter/storehaus_internal/offline", + "src/scala/com/twitter/storehaus_internal/util", + "src/scala/com/twitter/topic_recos/stores", + "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", + "src/thrift/com/twitter/manhattan:v1-scala", + "src/thrift/com/twitter/topic_recos:topic_recos-thrift-scala", + "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", + "timelines/src/main/scala/com/twitter/timelines/clients/ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/store", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/CertoStratoTopicTweetsStoreModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/CertoStratoTopicTweetsStoreModule.scala new file mode 100644 index 000000000..c7f881660 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/CertoStratoTopicTweetsStoreModule.scala @@ -0,0 +1,58 @@ +package com.twitter.tweet_mixer.module + +import com.twitter.inject.TwitterModule +import com.twitter.finagle.memcached.{Client => MemcachedClient} +import com.twitter.storehaus.ReadableStore +import com.twitter.tweet_mixer.model.ModuleNames +import com.google.inject.Provides +import com.google.inject.Singleton +import com.google.inject.name.Named +import com.twitter.conversions.DurationOps._ +import com.twitter.hashing.KeyHasher +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hermit.store.common.ObservedCachedReadableStore +import com.twitter.hermit.store.common.ObservedMemcachedReadableStore +import com.twitter.hermit.store.common.ObservedReadableStore +import com.twitter.relevance_platform.common.injection.LZ4Injection +import com.twitter.relevance_platform.common.injection.SeqObjectInjection +import com.twitter.simclusters_v2.thriftscala.TopicId +import com.twitter.strato.client.Client +import com.twitter.topic_recos.stores.CertoTopicTopKTweetsStore +import com.twitter.topic_recos.thriftscala.TweetWithScores + +object CertoStratoTopicTweetsStoreModule extends TwitterModule { + + @Provides + @Singleton + @Named(ModuleNames.CertoStratoTopicTweetsStoreName) + def providesCertoStratoTopicTweetsStore( + memcachedClient: MemcachedClient, + stratoClient: Client, + statsReceiver: StatsReceiver + ): ReadableStore[TopicId, Seq[TweetWithScores]] = { + val certoStore = ObservedReadableStore(CertoTopicTopKTweetsStore.prodStore(stratoClient))( + statsReceiver.scope(ModuleNames.CertoStratoTopicTweetsStoreName)).mapValues { + topKTweetsWithScores => + topKTweetsWithScores.topTweetsByFollowerL2NormalizedCosineSimilarityScore + } + + val memCachedStore = ObservedMemcachedReadableStore + .fromCacheClient( + backingStore = certoStore, + cacheClient = memcachedClient, + ttl = 10.minutes + )( + valueInjection = LZ4Injection.compose(SeqObjectInjection[TweetWithScores]()), + statsReceiver = statsReceiver.scope("memcached_certo_store"), + keyToString = { k => s"certo:${KeyHasher.FNV1A_64.hashKey(k.toString.getBytes)}" } + ) + + ObservedCachedReadableStore.from[TopicId, Seq[TweetWithScores]]( + memCachedStore, + ttl = 5.minutes, + maxKeys = 100000, // ~150MB max + cacheName = "certo_in_memory_cache", + windowSize = 10000L + )(statsReceiver.scope("certo_in_memory_cache")) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/ExtendedStratoClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/ExtendedStratoClientModule.scala new file mode 100644 index 000000000..346fc851d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/ExtendedStratoClientModule.scala @@ -0,0 +1,70 @@ +package com.twitter.tweet_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.service.Retries +import com.twitter.finagle.service.RetryPolicy +import com.twitter.finagle.ssl.OpportunisticTls +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.inject.TwitterModule +import com.twitter.strato.client.Client +import com.twitter.strato.client.Strato +import com.twitter.util.Try +import javax.inject.Named +import javax.inject.Singleton + +/** + * Strato Finagle config is copied from TLX + */ +object ExtendedStratoClientModule extends TwitterModule { + + private val StratoClientConnectionTimeout = 200.millis + private val StratoClientAcquisitionTimeout = 500.millis + private val StandardStratoClientRequestTimeout = 280.millis + private val ModerateStratoClientRequestTimeout = 500.millis + + private val DefaultRetryPartialFunction: PartialFunction[Try[Nothing], Boolean] = + RetryPolicy.TimeoutAndWriteExceptionsOnly + .orElse(RetryPolicy.ChannelClosedExceptionsOnly) + + protected def mkRetryPolicy(tries: Int): RetryPolicy[Try[Nothing]] = + RetryPolicy.tries(tries, DefaultRetryPartialFunction) + + @Singleton + @Provides + @Named("StratoClientWithDefaultTimeout") + def providesDefaultStratoClient( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): Client = { + Strato.client + .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) + .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) + .withTransport.connectTimeout(StratoClientConnectionTimeout) + .withRequestTimeout(StandardStratoClientRequestTimeout) + .withPerRequestTimeout(StandardStratoClientRequestTimeout) + .configured(Retries.Policy(mkRetryPolicy(1))) + .withStatsReceiver(statsReceiver.scope("default_strato_client")) + .build() + } + + @Singleton + @Provides + @Named("StratoClientWithModerateTimeout") + def providesModerateStratoClient( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): Client = { + Strato.client + .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) + .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) + .withTransport.connectTimeout(StratoClientConnectionTimeout) + .withRequestTimeout(ModerateStratoClientRequestTimeout) + .withPerRequestTimeout(ModerateStratoClientRequestTimeout) + .withRpcBatchSize(5) + .configured(Retries.Policy(mkRetryPolicy(1))) + .withStatsReceiver(statsReceiver.scope("moderate_strato_client")) + .build() + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/GPURetrievalHttpClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/GPURetrievalHttpClientModule.scala new file mode 100644 index 000000000..eeb3b2e97 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/GPURetrievalHttpClientModule.scala @@ -0,0 +1,138 @@ +package com.twitter.tweet_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.conversions.StorageUnitOps.richStorageUnitFromInt +import com.twitter.finagle._ +import com.twitter.finagle.builder.ClientBuilder +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.DefaultStatsReceiver +import com.twitter.finagle.stats.NullStatsReceiver +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.tracing.DefaultTracer +import com.twitter.finagle.tracing.Tracer +import com.twitter.finagle.Service +import com.twitter.finagle.http.Fields +import com.twitter.finagle.http.Method +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Status +import com.twitter.finagle.http.Version +import com.twitter.inject.TwitterModule +import com.twitter.io.Buf +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.util.Future +import com.twitter.util.Duration +import com.twitter.util.jackson.JSON +import javax.inject.Named +import javax.inject.Singleton + +object GPURetrievalHttpClientModule extends TwitterModule { + + @Provides + @Singleton + @Named(ModuleNames.GPURetrievalProdHttpClient) + def providesGPURetrievalProdHttpClientModule( + serviceIdentifier: ServiceIdentifier + ): GPURetrievalHttpClient = { + + GPURetrievalHttpClientBuilder(serviceIdentifier = serviceIdentifier).build() + } + + @Provides + @Singleton + @Named(ModuleNames.GPURetrievalDevelHttpClient) + def providesGPURetrievalDevelHttpClientModule( + serviceIdentifier: ServiceIdentifier + ): GPURetrievalHttpClient = { + + GPURetrievalHttpClientBuilder( + environment = "devel", + serviceIdentifier = serviceIdentifier + ).build() + } +} + +class GPURetrievalHttpClient( + client: Service[Request, Response], + baseStatsReceiver: StatsReceiver) { + + import GPURetrievalHttpClient._ + + val statsReceiver = baseStatsReceiver.scope("GPURetrievalHttpClientModule") + + def getNeighbors( + embeddings: Seq[Int], + ): Future[Seq[(Long, Double)]] = { + + val payload = Payload(embeddings) + val jsonInput = JSON.write(payload) + + val httpResponseFut = + postJsonRequest("/inference", jsonInput) + .flatMap(response => { + if (response.status == Status.Ok) { + statsReceiver.counter("StatusOk").incr + Future.value(response.contentString) + } else { + statsReceiver.counter("StatusError").incr + Future.exception( + new Exception(s"${response.statusCode} Error: ${response.contentString}")) + } + }) + httpResponseFut.map { response => + val responseOpt: Option[Result] = JSON.parse[Result](response) + responseOpt.getOrElse(Result(Seq.empty)).result + } + } + + private def postJsonRequest(path: String, payload: String): Future[Response] = { + val req = Request(Version.Http11, Method.Post, path) + val headers = req.headerMap + headers.set(Fields.ContentType, "application/json") + req.content = Buf.Utf8(payload) + req.headerMap.set(Fields.ContentLength, payload.length.toString) + // This is required by the server due to strict HTTP/1.1 impl. Otherwise 400 invalid request will be thrown + req.headerMap.set("Host", "") + client(req) + } +} + +case class GPURetrievalHttpClientBuilder( + name: String = "gpu-retrieval", + role: String = "gpu-retrieval", + host: String = "gpu-retrieval", + environment: String = "prod", + maxRequestPerSec: Option[Int] = None, + hostConnectionLimit: Int = 10, + requestTimeout: Duration = 300.millis, + serviceIdentifier: ServiceIdentifier, + tracer: Tracer = DefaultTracer, + statsReceiver: StatsReceiver = DefaultStatsReceiver, + hostStatsReceiver: StatsReceiver = new NullStatsReceiver(), + numOfRetries: Int = 3) { + + def withRequestTimeout(requestTimeout: Duration) = + this.copy(requestTimeout = requestTimeout) + + def build(): GPURetrievalHttpClient = { + val dest = if (environment == "prod") s"/s/$role/$host" else s"/srv#/devel/local/$role/$host" + val builder = ClientBuilder() + .name(name) + .dest(dest) + .reportTo(this.statsReceiver) + .reportHostStats(this.hostStatsReceiver) + .hostConnectionLimit(this.hostConnectionLimit) + .stack(Http.client.withMaxResponseSize(512.megabytes)) + .requestTimeout(this.requestTimeout) + .retries(this.numOfRetries) + .tracer(this.tracer) + + new GPURetrievalHttpClient(builder.build(), statsReceiver) + } +} + +object GPURetrievalHttpClient { + case class Payload(embeddings: Seq[Int]) + case class Result(result: Seq[(Long, Double)]) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HaploliteClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HaploliteClientModule.scala new file mode 100644 index 000000000..cc6276cec --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HaploliteClientModule.scala @@ -0,0 +1,24 @@ +package com.twitter.tweet_mixer.module + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.thriftmux.MethodBuilder +import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient +import com.twitter.haplolite.{thriftscala => t} +import com.twitter.inject.Injector +import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule + +object HaploliteClientModule + extends ThriftMethodBuilderClientModule[ + t.Haplolite.ServicePerEndpoint, + t.Haplolite.MethodPerEndpoint + ] + with MtlsClient { + override def label: String = "haplolite" + + override def dest: String = "/s/haplolite/haplolite-nonfanout" + + override protected def configureMethodBuilder( + injector: Injector, + methodBuilder: MethodBuilder + ): MethodBuilder = methodBuilder.withTimeoutTotal(100.milliseconds) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HydraEmbeddingGenerationServiceClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HydraEmbeddingGenerationServiceClientModule.scala new file mode 100644 index 000000000..c06e559ef --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HydraEmbeddingGenerationServiceClientModule.scala @@ -0,0 +1,41 @@ +package com.twitter.tweet_mixer.module + +import com.twitter.conversions.DurationOps._ +import com.twitter.conversions.PercentOps._ +import com.twitter.finagle.thriftmux.MethodBuilder +import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient +import com.twitter.hydra.embedding_generation.thriftscala.EmbeddingGenerationService +import com.twitter.inject.Injector +import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule +import com.twitter.util.Duration + +object HydraEmbeddingGenerationServiceClientModule + extends ThriftMethodBuilderClientModule[ + EmbeddingGenerationService.ServicePerEndpoint, + EmbeddingGenerationService.MethodPerEndpoint + ] + with MtlsClient { + + final val PerRequestTimeout = flag[Duration]( + name = "egs.per_request_timeout", + default = 150.millis, + help = "Timeout for each request to EGS in millisecond") + + final val TotalTimeout = flag[Duration]( + name = "egs.total_timeout", + default = 250.millis, + help = "Total timeout for all requests to EGS in millisecond") + + override val label: String = "embedding-generation-service" + override val dest: String = "/s/hydra/hydra-embedding-generation" + + override protected def configureMethodBuilder( + injector: Injector, + methodBuilder: MethodBuilder + ): MethodBuilder = { + methodBuilder + .withTimeoutPerRequest(PerRequestTimeout()) + .withTimeoutTotal(TotalTimeout()) + .idempotent(1.percent) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HydraRootClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HydraRootClientModule.scala new file mode 100644 index 000000000..491c3f1db --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/HydraRootClientModule.scala @@ -0,0 +1,24 @@ +package com.twitter.tweet_mixer.module + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.thriftmux.MethodBuilder +import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient +import com.twitter.hydra.root.{thriftscala => ht} +import com.twitter.inject.Injector +import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule + +object HydraRootClientModule + extends ThriftMethodBuilderClientModule[ + ht.HydraRoot.ServicePerEndpoint, + ht.HydraRoot.MethodPerEndpoint + ] + with MtlsClient { + override def label: String = "hydra-root" + + override def dest: String = "/s/hydra/hydra-root" + + override protected def configureMethodBuilder( + injector: Injector, + methodBuilder: MethodBuilder + ): MethodBuilder = methodBuilder.withTimeoutTotal(500.milliseconds) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/InMemoryCacheModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/InMemoryCacheModule.scala new file mode 100644 index 000000000..e5d52fdba --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/InMemoryCacheModule.scala @@ -0,0 +1,68 @@ +package com.twitter.tweet_mixer.module + +import com.github.benmanes.caffeine.cache.Caffeine +import com.google.inject.Provides +import com.twitter.inject.TwitterModule +import com.twitter.stitch.Stitch +import com.twitter.stitch.cache.AsyncValueCache +import com.twitter.stitch.cache.ConcurrentMapCache +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.utils.Utils + +import java.util.concurrent.ConcurrentMap +import java.util.concurrent.TimeUnit +import javax.inject.Named +import javax.inject.Singleton + +object InMemoryCacheModule extends TwitterModule { + @Singleton + @Provides + @Named(ModuleNames.TweeypieInMemCache) + def providesTweetypieInMemCache( + ): AsyncValueCache[java.lang.Long, Option[(Int, Long, Long)]] = { + val TTLSeconds = Utils.randomizedTTL(15 * 60) + val mapCache: ConcurrentMap[java.lang.Long, Stitch[Option[(Int, Long, Long)]]] = + Caffeine + .newBuilder() + .recordStats() + .expireAfterWrite(TTLSeconds, TimeUnit.SECONDS) + .maximumSize(8388607) + .build[java.lang.Long, Stitch[Option[(Int, Long, Long)]]] + .asMap + AsyncValueCache.apply(new ConcurrentMapCache(mapCache)) + } + + @Singleton + @Provides + @Named(ModuleNames.MediaMetadataInMemCache) + def providesMediaMetadataInMemCache( + ): AsyncValueCache[java.lang.Long, Option[Long]] = { + val TTLSeconds = Utils.randomizedTTL(15 * 60) + val mapCache: ConcurrentMap[java.lang.Long, Stitch[Option[Long]]] = + Caffeine + .newBuilder() + .recordStats() + .expireAfterWrite(TTLSeconds, TimeUnit.SECONDS) + .maximumSize(8388607) + .build[java.lang.Long, Stitch[Option[Long]]] + .asMap + AsyncValueCache.apply(new ConcurrentMapCache(mapCache)) + } + + @Singleton + @Provides + @Named(ModuleNames.GrokFilterInMemCache) + def providesGrokFilterInMemCache( + ): AsyncValueCache[java.lang.Long, Boolean] = { + val TTLSeconds = Utils.randomizedTTL(15 * 60) + val mapCache: ConcurrentMap[java.lang.Long, Stitch[Boolean]] = + Caffeine + .newBuilder() + .recordStats() + .expireAfterWrite(TTLSeconds, TimeUnit.SECONDS) + .maximumSize(8388607) + .build[java.lang.Long, Stitch[Boolean]] + .asMap + AsyncValueCache.apply(new ConcurrentMapCache(mapCache)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/MHMtlsParamsModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/MHMtlsParamsModule.scala new file mode 100644 index 000000000..8f5d2dea5 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/MHMtlsParamsModule.scala @@ -0,0 +1,17 @@ +package com.twitter.tweet_mixer.module + +import com.google.inject.Provides +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.inject.TwitterModule +import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams +import javax.inject.Singleton + +object MHMtlsParamsModule extends TwitterModule { + @Singleton + @Provides + def providesManhattanMtlsParams( + serviceIdentifier: ServiceIdentifier + ): ManhattanKVClientMtlsParams = { + ManhattanKVClientMtlsParams(serviceIdentifier) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/ManhattanFeatureRepositoryModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/ManhattanFeatureRepositoryModule.scala new file mode 100644 index 000000000..2f39e231b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/ManhattanFeatureRepositoryModule.scala @@ -0,0 +1,82 @@ +package com.twitter.tweet_mixer.module + +import com.google.inject.Provides +import com.twitter.bijection.Injection +import com.twitter.bijection.scrooge.BinaryScalaCodec +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.inject.TwitterModule +import com.twitter.manhattan.v1.{thriftscala => mh} +import com.twitter.product_mixer.shared_library.manhattan_client.ManhattanClientBuilder +import com.twitter.servo.manhattan.ManhattanKeyValueRepository +import com.twitter.servo.repository.KeyValueRepository +import com.twitter.servo.repository.Repository +import com.twitter.storehaus_internal.manhattan.ManhattanClusters +import com.twitter.tweet_mixer.model.ModuleNames._ +import com.twitter.tweet_mixer.utils.InjectionTransformerImplicits._ +import com.twitter.wtf.candidate.thriftscala.CandidateSeq +import java.nio.ByteBuffer +import javax.inject.Named +import javax.inject.Singleton +import org.apache.thrift.transport.TMemoryInputTransport +import org.apache.thrift.transport.TTransport + +object ManhattanFeatureRepositoryModule extends TwitterModule { + + @Provides + @Singleton + @Named(ManhattanAthenaClient) + def providesManhattanAthenaClient( + serviceIdentifier: ServiceIdentifier + ): mh.ManhattanCoordinator.MethodPerEndpoint = { + ManhattanClientBuilder + .buildManhattanV1FinagleClient( + ManhattanClusters.athena, + serviceIdentifier + ) + } + + @Provides + @Singleton + @Named(ManhattanApolloClient) + def providesManhattanApolloClient( + serviceIdentifier: ServiceIdentifier + ): mh.ManhattanCoordinator.MethodPerEndpoint = { + ManhattanClientBuilder + .buildManhattanV1FinagleClient( + ManhattanClusters.apollo, + serviceIdentifier + ) + } + + @Provides + @Singleton + @Named(RealGraphInNetworkScoresOnPremRepo) + def providesRealGraphInNetworkScoresRepo( + @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint + ): Repository[Long, Option[CandidateSeq]] = { + val keyTransformer = Injection + .connect[Long, Array[Byte]] + .toByteBufferTransformer() + + val valueTransformer = + BinaryScalaCodec(CandidateSeq).toByteBufferTransformer().flip + + KeyValueRepository.singular( + new ManhattanKeyValueRepository( + client = client, + keyTransformer = keyTransformer, + valueTransformer = valueTransformer, + appId = "real_graph", + dataset = "real_graph_scores_in_v1", + timeoutInMillis = 100 + ) + ) + } + + private def transportFromByteBuffer(buffer: ByteBuffer): TTransport = + new TMemoryInputTransport( + buffer.array(), + buffer.arrayOffset() + buffer.position(), + buffer.remaining() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/MemCacheClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/MemCacheClientModule.scala new file mode 100644 index 000000000..fa9370c4c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/MemCacheClientModule.scala @@ -0,0 +1,34 @@ +package com.twitter.tweet_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.memcached.{Client => MemcachedClient} +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hashing.KeyHasher +import com.twitter.inject.TwitterModule +import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder +import javax.inject.Singleton + +object MemcacheClientModule extends TwitterModule { + + @Singleton + @Provides + def providesCache( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): MemcachedClient = { + MemcachedClientBuilder.buildMemcachedClient( + destName = "/srv#/prod/local/cache/content_recommender_unified_v2:twemcaches", + numTries = 1, + requestTimeout = 15.milliseconds, + globalTimeout = 20.milliseconds, + connectTimeout = 15.milliseconds, + acquisitionTimeout = 15.milliseconds, + serviceIdentifier = serviceIdentifier, + statsReceiver = statsReceiver, + failureAccrualPolicy = None, + keyHasher = Some(KeyHasher.FNV1A_64) + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/PipelineFailureExceptionMapper.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/PipelineFailureExceptionMapper.scala new file mode 100644 index 000000000..d3bdbfab6 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/PipelineFailureExceptionMapper.scala @@ -0,0 +1,29 @@ +package com.twitter.tweet_mixer.module + +import com.twitter.finatra.thrift.exceptions.ExceptionMapper +import com.twitter.inject.Logging +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled +import com.twitter.tweet_mixer.{thriftscala => t} +import com.twitter.scrooge.ThriftException +import com.twitter.util.Future +import javax.inject.Singleton + +@Singleton +class PipelineFailureExceptionMapper + extends ExceptionMapper[PipelineFailure, ThriftException] + with Logging { + + def handleException(throwable: PipelineFailure): Future[ThriftException] = { + throwable match { + // SliceService (unlike UrtService) throws an exception when the requested product is disabled + case PipelineFailure(ProductDisabled, reason, _, _) => + Future.exception( + t.ValidationExceptionList(errors = + Seq(t.ValidationException(t.ValidationErrorCode.ProductDisabled, reason)))) + case _ => + error("Unhandled PipelineFailure", throwable) + Future.exception(throwable) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SampleFeatureStoreV1DynamicClientBuilderModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SampleFeatureStoreV1DynamicClientBuilderModule.scala new file mode 100644 index 000000000..1b3a3d835 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SampleFeatureStoreV1DynamicClientBuilderModule.scala @@ -0,0 +1,50 @@ +package com.twitter.tweet_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.inject.TwitterModule +import com.twitter.ml.featurestore.lib.dataset.DatasetParams +import com.twitter.ml.featurestore.lib.dynamic.BaseDynamicHydrationConfig +import com.twitter.ml.featurestore.lib.dynamic.BaseGatedFeatures +import com.twitter.ml.featurestore.lib.dynamic.ClientConfig +import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient +import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig +import com.twitter.ml.featurestore.lib.params.FeatureStoreParams +import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1DynamicClientBuilder +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.strato.opcontext.Attribution.ManhattanAppId +import javax.inject.Singleton + +object SampleFeatureStoreV1DynamicClientBuilderModule extends TwitterModule { + + @Provides + @Singleton + def provideFeatureStoreV1DynamicClientBuilder( + statsReceiver: StatsReceiver, + serviceIdentifier: ServiceIdentifier + ): FeatureStoreV1DynamicClientBuilder = { + val defaultFeatureStoreParams = FeatureStoreParams(global = DatasetParams( + attributions = Seq(ManhattanAppId("athena", "explore_mixer_features_athena")), + logHydrationErrors = true)) + + new FeatureStoreV1DynamicClientBuilder { + override def build[Query <: PipelineQuery]( + dynamicHydrationConfig: BaseDynamicHydrationConfig[Query, _ <: BaseGatedFeatures[Query]] + ): DynamicFeatureStoreClient[Query] = + DynamicFeatureStoreClient( + clientConfig = ClientConfig( + dynamicHydrationConfig = dynamicHydrationConfig, + featureStoreParamsConfig = + FeatureStoreParamsConfig(defaultFeatureStoreParams = defaultFeatureStoreParams), + timeoutProvider = _ => 350.milliseconds, + timeoutPerRequestProvider = None, + stratoMaxBatchSizeProvider = None, + serviceIdentifier = serviceIdentifier + ), + statsReceiver = statsReceiver + ) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SimClustersANNServiceNameToClientMapper.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SimClustersANNServiceNameToClientMapper.scala new file mode 100644 index 000000000..b4e073c6a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SimClustersANNServiceNameToClientMapper.scala @@ -0,0 +1,23 @@ +package com.twitter.tweet_mixer.module + +import com.google.inject.Provides +import com.google.inject.Singleton +import com.twitter.inject.TwitterModule +import com.twitter.simclustersann.thriftscala.SimClustersANNService +import com.twitter.tweet_mixer.model.ModuleNames +import javax.inject.Named + +object SimClustersANNServiceNameToClientMapper extends TwitterModule { + + @Provides + @Singleton + def providesSimClustersANNServiceNameToClientMapping( + @Named(ModuleNames.ProdSimClustersANNServiceClientName) simClustersANNServiceProd: SimClustersANNService.MethodPerEndpoint, + @Named(ModuleNames.SimClustersVideoANNServiceClientName) simClustersVideoANNServiceProd: SimClustersANNService.MethodPerEndpoint + ): Map[String, SimClustersANNService.MethodPerEndpoint] = { + Map[String, SimClustersANNService.MethodPerEndpoint]( + "simclusters-ann" -> simClustersANNServiceProd, + "simclusters-ann-experimental" -> simClustersVideoANNServiceProd + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SkitStratoTopicTweetsStoreModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SkitStratoTopicTweetsStoreModule.scala new file mode 100644 index 000000000..b306d2127 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/SkitStratoTopicTweetsStoreModule.scala @@ -0,0 +1,62 @@ +package com.twitter.tweet_mixer.module + +import com.twitter.inject.TwitterModule +import com.twitter.finagle.memcached.{Client => MemcachedClient} +import com.twitter.storehaus.ReadableStore +import com.twitter.tweet_mixer.model.ModuleNames +import com.google.inject.Provides +import com.google.inject.Singleton +import com.google.inject.name.Named +import com.twitter.conversions.DurationOps._ +import com.twitter.frigate.common.store.strato.StratoFetchableStore +import com.twitter.hashing.KeyHasher +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hermit.store.common.ObservedCachedReadableStore +import com.twitter.hermit.store.common.ObservedMemcachedReadableStore +import com.twitter.hermit.store.common.ObservedReadableStore +import com.twitter.relevance_platform.common.injection.LZ4Injection +import com.twitter.relevance_platform.common.injection.SeqObjectInjection +import com.twitter.strato.client.Client +import com.twitter.topic_recos.thriftscala.TopicTopTweets +import com.twitter.topic_recos.thriftscala.TopicTweet +import com.twitter.topic_recos.thriftscala.TopicTweetPartitionFlatKey + +object SkitStratoTopicTweetsStoreModule extends TwitterModule { + + final val column = "recommendations/topic_recos/topicTopTweets" + + @Provides + @Singleton + @Named(ModuleNames.SkitStratoTopicTweetsStoreName) + def providesSkitStratoTopicTweetsStore( + memcachedClient: MemcachedClient, + stratoClient: Client, + statsReceiver: StatsReceiver + ): ReadableStore[TopicTweetPartitionFlatKey, Seq[TopicTweet]] = { + val skitStore = ObservedReadableStore( + StratoFetchableStore + .withUnitView[TopicTweetPartitionFlatKey, TopicTopTweets](stratoClient, column))( + statsReceiver.scope(ModuleNames.SkitStratoTopicTweetsStoreName)).mapValues { topicTopTweets => + topicTopTweets.topTweets + } + + val memCachedStore = ObservedMemcachedReadableStore + .fromCacheClient( + backingStore = skitStore, + cacheClient = memcachedClient, + ttl = 10.minutes + )( + valueInjection = LZ4Injection.compose(SeqObjectInjection[TopicTweet]()), + statsReceiver = statsReceiver.scope("memcached_skit_store"), + keyToString = { k => s"skit:${KeyHasher.FNV1A_64.hashKey(k.toString.getBytes)}" } + ) + + ObservedCachedReadableStore.from[TopicTweetPartitionFlatKey, Seq[TopicTweet]]( + memCachedStore, + ttl = 5.minutes, + maxKeys = 100000, // ~150MB max + cacheName = "skit_in_memory_cache", + windowSize = 10000L + )(statsReceiver.scope("skit_in_memory_cache")) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/StitchMemcacheClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/StitchMemcacheClientModule.scala new file mode 100644 index 000000000..23b9837aa --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/StitchMemcacheClientModule.scala @@ -0,0 +1,20 @@ +package com.twitter.tweet_mixer.module + +import com.google.inject.Provides +import com.twitter.finagle.memcached.{Client => MemcachedClient} +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.inject.TwitterModule +import com.twitter.tweet_mixer.utils.MemcacheStitchClient +import javax.inject.Singleton + +object StitchMemcacheClientModule extends TwitterModule { + + @Provides + @Singleton + def providesMemcacheStitchClient( + memcacheClient: MemcachedClient, + statsReceiver: StatsReceiver + ): MemcacheStitchClient = { + new MemcacheStitchClient(memcacheClient, statsReceiver) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TimeoutConfigModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TimeoutConfigModule.scala new file mode 100644 index 000000000..44c5b13bc --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TimeoutConfigModule.scala @@ -0,0 +1,23 @@ +package com.twitter.tweet_mixer.module + +import com.twitter.conversions.DurationOps._ +import com.google.inject.Provides +import com.twitter.inject.TwitterModule +import com.twitter.tweet_mixer.config.TimeoutConfig +import javax.inject.Singleton + +object TimeoutConfigModule extends TwitterModule { + @Provides + @Singleton + def provideTimeoutBudget(): TimeoutConfig = { + TimeoutConfig( + thriftAnnServiceClientTimeout = 50.milliseconds, + thriftSANNServiceClientTimeout = 50.milliseconds, + thriftTweetypieClientTimeout = 50.milliseconds, + thriftUserTweetGraphClientTimeout = 50.milliseconds, + thriftUserVideoGraphClientTimeout = 50.milliseconds, + candidateSourceTimeout = 100.milliseconds, + userStateStoreTimeout = 12.milliseconds + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TwHINANNServiceModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TwHINANNServiceModule.scala new file mode 100644 index 000000000..95d557815 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TwHINANNServiceModule.scala @@ -0,0 +1,33 @@ +package com.twitter.tweet_mixer.module + +import com.google.inject.Provides +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.inject.TwitterModule +import com.twitter.timelines.clients.ann.ANNQdrantGRPCClient +import com.twitter.timelines.clients.ann.ANNQdrantService +import com.twitter.tweet_mixer.model.ModuleNames + +import javax.inject.Named +import javax.inject.Singleton + +object TwHINANNServiceModule extends TwitterModule { + + @Provides + @Singleton + @Named(ModuleNames.TwHINANNServiceClient) + def providesTwHINANNServiceClient( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver, + ): ANNQdrantGRPCClient = { + val dest = s"/s/ml-serving/twhin-serving-qdrant" + + new ANNQdrantGRPCClient( + new ANNQdrantService( + dest, + Some(serviceIdentifier) + ), + statsReceiver + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TwHINEmbeddingStoreModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TwHINEmbeddingStoreModule.scala new file mode 100644 index 000000000..8437c8a3c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TwHINEmbeddingStoreModule.scala @@ -0,0 +1,54 @@ +package com.twitter.tweet_mixer.module + +import com.google.inject.Provides +import com.twitter.frigate.common.store.strato.StratoFetchableStore +import com.twitter.tweet_mixer.store.TwhinEmbeddingsStore +import com.twitter.inject.TwitterModule +import com.twitter.simclusters_v2.common.TweetId +import com.twitter.simclusters_v2.common.VersionId +import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding +import com.twitter.storehaus.ReadableStore +import com.twitter.strato.client.{Client => StratoClient} +import com.twitter.tweet_mixer.model.ModuleNames + +import javax.inject.Named +import javax.inject.Singleton + +object TwHINEmbeddingStoreModule extends TwitterModule { + + val prodCachedTweetStratoColumn = "recommendations/twhin/CachedTwhinTweetEmbeddings.Tweet" + val prodCachedRebuildTweetStratoColumn = + "recommendations/twhin/CachedTwhinRebuildVersionedTweetEmbeddings" + + @Provides + @Singleton + @Named(ModuleNames.TwhinRebuildUserPositiveEmbeddingsStore) + def providesTwhinRebuildUserPositiveEmbeddingFeatureStore( + twhinEmbeddingStore: TwhinEmbeddingsStore + ): ReadableStore[(Long, Long), TwhinTweetEmbedding] = + twhinEmbeddingStore.mhRebuildUserPositiveStore + + @Provides + @Singleton + @Named(ModuleNames.TwHINTweetEmbeddingStratoStore) + def providesTwhinTweetEmbeddingStore( + stratoClient: StratoClient + ): ReadableStore[TweetId, TwhinTweetEmbedding] = { + StratoFetchableStore.withUnitView[TweetId, TwhinTweetEmbedding]( + stratoClient, + column = prodCachedTweetStratoColumn + ) + } + + @Provides + @Singleton + @Named(ModuleNames.TwHINRebuildTweetEmbeddingStratoStore) + def providesTwHINRebuildTweetEmbeddingStore( + stratoClient: StratoClient + ): ReadableStore[(TweetId, VersionId), TwhinTweetEmbedding] = { + StratoFetchableStore.withUnitView[(TweetId, VersionId), TwhinTweetEmbedding]( + stratoClient, + column = prodCachedRebuildTweetStratoColumn + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TweetMixerFlagModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TweetMixerFlagModule.scala new file mode 100644 index 000000000..b8d2ef07b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/TweetMixerFlagModule.scala @@ -0,0 +1,17 @@ +package com.twitter.tweet_mixer.module + +import com.twitter.inject.TwitterModule + +object TweetMixerFlagName { + final val DarkTrafficFilterDeciderKey = "thrift.dark.traffic.filter.decider_key" +} + +object TweetMixerFlagModule extends TwitterModule { + import TweetMixerFlagName._ + + flag[String]( + name = DarkTrafficFilterDeciderKey, + default = "enable_dark_traffic_filter", + help = "Dark traffic filter decider key" + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/UserStateStoreModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/UserStateStoreModule.scala new file mode 100644 index 000000000..fdfbf841c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/UserStateStoreModule.scala @@ -0,0 +1,101 @@ +package com.twitter.tweet_mixer.module + +import com.google.inject.Provides +import com.google.inject.Singleton +import com.twitter.bijection.Bufferable +import com.twitter.bijection.Injection +import com.twitter.bijection.scrooge.BinaryScalaCodec +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.memcached.{Client => MemcachedClient} +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hermit.store.common.ObservedMemcachedReadableStore +import com.twitter.inject.TwitterModule +import com.twitter.simclusters_v2.common.UserId +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams +import com.twitter.storehaus.ReadableStore +import com.twitter.storehaus_internal.manhattan.ManhattanRO +import com.twitter.storehaus_internal.manhattan.ManhattanROConfig +import com.twitter.storehaus_internal.util.HDFSPath +import com.twitter.core_workflows.user_model.thriftscala.UserState +import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState +import com.twitter.tweet_mixer.config.TimeoutConfig +import com.twitter.storehaus_internal.manhattan.Apollo +import com.twitter.storehaus_internal.util.ApplicationID +import com.twitter.storehaus_internal.util.DatasetName +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.util.JavaTimer +import com.twitter.util.Time +import com.twitter.util.TimeoutException +import com.twitter.util.Timer + +object UserStateStoreModule extends TwitterModule { + implicit val timer: Timer = new JavaTimer(true) + final val NewUserCreateDaysThreshold = 7 + final val DefaultUnknownUserStateValue = 100 + + // Convert CondensedUserState to UserState Enum + // If CondensedUserState is None, back fill by checking whether the user is new user + class UserStateStore( + userStateStore: ReadableStore[UserId, CondensedUserState], + timeout: Duration, + statsReceiver: StatsReceiver) + extends ReadableStore[UserId, UserState] { + override def get(userId: UserId): Future[Option[UserState]] = { + userStateStore + .get(userId).map(_.flatMap(_.userState)).map { + case Some(userState) => Some(userState) + case None => + val isNewUser = SnowflakeId.timeFromIdOpt(userId).exists { userCreateTime => + Time.now - userCreateTime < Duration.fromDays(NewUserCreateDaysThreshold) + } + if (isNewUser) Some(UserState.New) + else Some(UserState.EnumUnknownUserState(DefaultUnknownUserStateValue)) + + }.raiseWithin(timeout)(timer).rescue { + case _: TimeoutException => + statsReceiver.counter("TimeoutException").incr() + Future.None + } + } + } + + @Provides + @Singleton + def providesUserStateStore( + statsReceiver: StatsReceiver, + manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, + memcachedClient: MemcachedClient, + timeoutConfig: TimeoutConfig + ): ReadableStore[UserId, UserState] = { + + val underlyingStore = new UserStateStore( + ManhattanRO + .getReadableStoreWithMtls[UserId, CondensedUserState]( + ManhattanROConfig( + HDFSPath(""), + ApplicationID("cr_mixer_apollo"), + DatasetName("condensed_user_state"), + Apollo), + manhattanKVClientMtlsParams + )( + implicitly[Injection[Long, Array[Byte]]], + BinaryScalaCodec(CondensedUserState) + ), + timeoutConfig.userStateStoreTimeout, + statsReceiver.scope("UserStateStore") + ).mapValues(_.value) // Read the value of Enum so that we only caches the Int + + ObservedMemcachedReadableStore + .fromCacheClient( + backingStore = underlyingStore, + cacheClient = memcachedClient, + ttl = 24.hours, + )( + valueInjection = Bufferable.injectionOf[Int], // Cache Value is Enum Value for UserState + statsReceiver = statsReceiver.scope("memCachedUserStateStore"), + keyToString = { k: UserId => s"uState/$k" } + ).mapValues(value => UserState.getOrUnknown(value)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnEmbeddingProducerModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnEmbeddingProducerModule.scala new file mode 100644 index 000000000..511939a2b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnEmbeddingProducerModule.scala @@ -0,0 +1,48 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.google.inject.Provides +import com.twitter.ann.common.EmbeddingProducer +import com.twitter.ann.manhattan.ManhattanEmbeddingProducer +import com.twitter.bijection.Injection +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.inject.TwitterModule +import com.twitter.storage.client.manhattan.kv.ManhattanKVClient +import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams +import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder +import com.twitter.tweet_mixer.model.ModuleNames +import javax.inject.Named +import javax.inject.Singleton + +object AnnEmbeddingProducerModule extends TwitterModule { + + @Provides + @Singleton + @Named(ModuleNames.TwHINEmbeddingRegularUpdateMhEmbeddingProducer) + def twHINEmbeddingRegularUpdateMhEmbeddingProducer( + serviceIdentifier: ServiceIdentifier, + ): EmbeddingProducer[Long] = { + val client = ManhattanKVClient( + "cr_mixer_apollo", + "/s/manhattan/apollo.native-thrift", + ManhattanKVClientMtlsParams(serviceIdentifier)) + val endpoint = ManhattanKVEndpointBuilder(client).defaultMaxTimeout(50.seconds).build() + val longCodec = implicitly[Injection[Long, Array[Byte]]] + ManhattanEmbeddingProducer("twhin_regular_update_tweet_embedding_apollo", longCodec, endpoint) + } + + @Provides + @Singleton + @Named(ModuleNames.ConsumerBasedTwHINEmbeddingRegularUpdateMhEmbeddingProducer) + def consumerBasedTwHINEmbeddingRegularUpdateMhEmbeddingProducer( + serviceIdentifier: ServiceIdentifier, + ): EmbeddingProducer[Long] = { + val client = ManhattanKVClient( + "cr_mixer_apollo", + "/s/manhattan/apollo.native-thrift", + ManhattanKVClientMtlsParams(serviceIdentifier)) + val endpoint = ManhattanKVEndpointBuilder(client).defaultMaxTimeout(50.seconds).build() + val longCodec = implicitly[Injection[Long, Array[Byte]]] + ManhattanEmbeddingProducer("twhin_user_embedding_regular_update_apollo", longCodec, endpoint) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnQueryServiceClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnQueryServiceClientModule.scala new file mode 100644 index 000000000..438c8fc73 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnQueryServiceClientModule.scala @@ -0,0 +1,58 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.google.inject.Provides +import com.twitter.ann.common.thriftscala.AnnQueryService +import com.twitter.conversions.DurationOps._ +import com.twitter.conversions.PercentOps._ +import com.twitter.tweet_mixer.config.TimeoutConfig +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.mtls.client.MtlsStackClient._ +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.inject.TwitterModule +import com.twitter.tweet_mixer.model.ModuleNames +import javax.inject.Named +import javax.inject.Singleton + +object AnnQueryServiceClientModule extends TwitterModule { + + @Provides + @Singleton + @Named(ModuleNames.TwHINRegularUpdateAnnServiceClientName) + def twHINRegularUpdateAnnServiceClient( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + statsReceiver: StatsReceiver, + timeoutConfig: TimeoutConfig, + ): AnnQueryService.MethodPerEndpoint = { + val dest = "/s/cassowary/twhin-regular-update-ann-service" + val label = "twhin_regular_update" + + buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) + } + + private def buildClient( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + timeoutConfig: TimeoutConfig, + statsReceiver: StatsReceiver, + dest: String, + label: String + ): AnnQueryService.MethodPerEndpoint = { + val thriftClient = ThriftMux.client + .withMutualTls(serviceIdentifier) + .withClientId(clientId) + .withLabel(label) + .withStatsReceiver(statsReceiver) + .withTransport.connectTimeout(500.milliseconds) + .withSession.acquisitionTimeout(500.milliseconds) + .methodBuilder(dest) + .withTimeoutPerRequest(timeoutConfig.thriftAnnServiceClientTimeout) + .withRetryDisabled + .idempotent(5.percent) + .servicePerEndpoint[AnnQueryService.ServicePerEndpoint] + + ThriftMux.Client.methodPerEndpoint(thriftClient) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnQueryableByIdModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnQueryableByIdModule.scala new file mode 100644 index 000000000..7f796c7c8 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/AnnQueryableByIdModule.scala @@ -0,0 +1,93 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.google.inject.Provides +import com.twitter.ann.common.AnnInjections +import com.twitter.ann.common.Cosine +import com.twitter.ann.common.CosineDistance +import com.twitter.ann.common.Distance +import com.twitter.ann.common.EmbeddingProducer +import com.twitter.ann.common.QueryableById +import com.twitter.ann.common.QueryableByIdImplementation +import com.twitter.ann.common.RuntimeParams +import com.twitter.ann.common.ServiceClientQueryable +import com.twitter.ann.common.thriftscala.AnnQueryService +import com.twitter.ann.common.thriftscala.NearestNeighborQuery +import com.twitter.ann.common.thriftscala.NearestNeighborResult +import com.twitter.ann.common.thriftscala.{Distance => ServiceDistance} +import com.twitter.ann.common.thriftscala.{RuntimeParams => ServiceRuntimeParams} +import com.twitter.ann.hnsw.HnswCommon +import com.twitter.ann.hnsw.HnswParams +import com.twitter.bijection.Injection +import com.twitter.finagle.Service +import com.twitter.inject.TwitterModule +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.util.Future +import javax.inject.Named +import javax.inject.Singleton + +object AnnQueryableByIdModule extends TwitterModule { + + @Provides + @Singleton + @Named(ModuleNames.TwHINAnnQueryableById) + def twHINAnnQueryableById( + @Named(ModuleNames.TwHINRegularUpdateAnnServiceClientName) + twhinAnnClient: AnnQueryService.MethodPerEndpoint, + @Named(ModuleNames.TwHINEmbeddingRegularUpdateMhEmbeddingProducer) + embeddingProducer: EmbeddingProducer[Long], + ): QueryableById[Long, Long, HnswParams, CosineDistance] = { + val runtimeParamInjection = HnswCommon.RuntimeParamsInjection + val distanceInjection = Cosine + val idInjection = AnnInjections.LongInjection + buildQueryableById( + twhinAnnClient, + embeddingProducer, + runtimeParamInjection, + distanceInjection, + idInjection) + } + + @Provides + @Singleton + @Named(ModuleNames.ConsumerBasedTwHINAnnQueryableById) + def consumerBasedTwHINAnnQueryableById( + @Named(ModuleNames.TwHINRegularUpdateAnnServiceClientName) + twhinAnnClient: AnnQueryService.MethodPerEndpoint, + @Named(ModuleNames.ConsumerBasedTwHINEmbeddingRegularUpdateMhEmbeddingProducer) + embeddingProducer: EmbeddingProducer[Long], + ): QueryableById[Long, Long, HnswParams, CosineDistance] = { + val runtimeParamInjection = HnswCommon.RuntimeParamsInjection + val distanceInjection = Cosine + val idInjection = AnnInjections.LongInjection + buildQueryableById( + twhinAnnClient, + embeddingProducer, + runtimeParamInjection, + distanceInjection, + idInjection) + } + + def buildQueryableById[T1, T2, P <: RuntimeParams, D <: Distance[D]]( + twhinAnnClient: AnnQueryService.MethodPerEndpoint, + embeddingProducer: EmbeddingProducer[T1], + runtimeParamInjection: Injection[P, ServiceRuntimeParams], + distanceInjection: Injection[D, ServiceDistance], + idInjection: Injection[T2, Array[Byte]] + ): QueryableById[T1, T2, P, D] = { + val service = new Service[NearestNeighborQuery, NearestNeighborResult] { + override def apply(request: NearestNeighborQuery): Future[NearestNeighborResult] = + twhinAnnClient.query(request) + } + + val queryableClient = new ServiceClientQueryable[T2, P, D]( + service, + runtimeParamInjection, + distanceInjection, + idInjection + ) + new QueryableByIdImplementation( + embeddingProducer, + queryableClient + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/BUILD.bazel new file mode 100644 index 000000000..e9a98b633 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/BUILD.bazel @@ -0,0 +1,42 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "ann/src/main/scala/com/twitter/ann/hnsw", + "ann/src/main/scala/com/twitter/ann/manhattan", + "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", + "decider/src/main/scala", + "finatra/inject/inject-thrift-client", + "geoduck/service/src/main/scala/com/twitter/geoduck/service/common/clientmodules", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", + "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client", + "qig-ranker/thrift/src/main/thrift:thrift-scala", + "relevance-platform/thrift/src/main/thrift:thrift-scala", + "servo/decider", + "simclusters-ann/thrift/src/main/thrift:thrift-scala", + "src/java/com/twitter/ml/api:api-base", + "src/java/com/twitter/search/queryparser/query:core-query-nodes", + "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", + "src/scala/com/twitter/algebird_internal/injection", + "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", + "src/scala/com/twitter/ml/api/embedding", + "src/scala/com/twitter/ml/featurestore/lib", + "src/scala/com/twitter/scalding_internal/multiformat/format", + "src/scala/com/twitter/storehaus_internal/manhattan", + "src/scala/com/twitter/storehaus_internal/manhattan/config", + "src/scala/com/twitter/storehaus_internal/memcache", + "src/scala/com/twitter/storehaus_internal/memcache/config", + "src/scala/com/twitter/storehaus_internal/offline", + "src/thrift/com/twitter/recos/user_tweet_graph:user_tweet_graph-scala", + "src/thrift/com/twitter/recos/user_video_graph:user_video_graph-scala", + "src/thrift/com/twitter/search:earlybird-scala", + "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", + "stitch/stitch-tweetypie/src/main/scala", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + "vecdb/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/EarlybirdRealtimeCGModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/EarlybirdRealtimeCGModule.scala new file mode 100644 index 000000000..8adbaf7f3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/EarlybirdRealtimeCGModule.scala @@ -0,0 +1,49 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ +import com.twitter.conversions.PercentOps._ +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.inject.TwitterModule +import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder +import com.twitter.product_mixer.shared_library.thrift_client.Idempotent +import com.twitter.search.earlybird.{thriftscala => t} +import com.twitter.tweet_mixer.model.ModuleNames.EarlybirdRealtimeCGEndpoint + +import javax.inject.Named +import javax.inject.Singleton +import org.apache.thrift.protocol.TCompactProtocol + +object EarlybirdRealtimeCGModule extends TwitterModule { + + val Label: String = "earlybird-rootrealtimecg" + val Dest: String = "/s/earlybird-rootrealtimecg/root-realtime_cg" + + @Provides + @Singleton + @Named(EarlybirdRealtimeCGEndpoint) + def providesEarlybirdRealtimeCGService( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + statsReceiver: StatsReceiver + ): t.EarlybirdService.MethodPerEndpoint = { + + FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ + t.EarlybirdService.ServicePerEndpoint, + t.EarlybirdService.MethodPerEndpoint + ]( + serviceIdentifier = serviceIdentifier, + clientId = clientId, + dest = Dest, + label = Label, + statsReceiver = statsReceiver, + protocolFactoryOverride = Some(new TCompactProtocol.Factory), + idempotency = Idempotent(1.percent), + timeoutPerRequest = 200.milliseconds, + timeoutTotal = 400.milliseconds, + acquisitionTimeout = 1.seconds + ) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/GeoduckHydrationClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/GeoduckHydrationClientModule.scala new file mode 100644 index 000000000..975ad540d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/GeoduckHydrationClientModule.scala @@ -0,0 +1,15 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.thrift.ClientId +import com.twitter.geoduck.service.common.clientmodules.HydrationThriftClientModule +import com.twitter.inject.Injector + +object GeoduckHydrationClientModule extends HydrationThriftClientModule { + override protected def configureThriftMuxClient( + injector: Injector, + client: ThriftMux.Client + ): ThriftMux.Client = { + client.withClientId(ClientId("tweet-mixer")) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/GeoduckLocationServiceClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/GeoduckLocationServiceClientModule.scala new file mode 100644 index 000000000..4ea89712d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/GeoduckLocationServiceClientModule.scala @@ -0,0 +1,15 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.thrift.ClientId +import com.twitter.geoduck.service.common.clientmodules.LocationServiceThriftClientModule +import com.twitter.inject.Injector + +object GeoduckLocationServiceClientModule extends LocationServiceThriftClientModule { + override protected def configureThriftMuxClient( + injector: Injector, + client: ThriftMux.Client + ): ThriftMux.Client = { + client.withClientId(ClientId("tweet-mixer")) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/QigServiceClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/QigServiceClientModule.scala new file mode 100644 index 000000000..ef8b54ed3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/QigServiceClientModule.scala @@ -0,0 +1,29 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.twitter.finagle.thriftmux.MethodBuilder +import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient +import com.twitter.inject.Injector +import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule +import com.twitter.search.query_interaction_graph.service.thriftscala.QigService +import com.twitter.conversions.DurationOps._ +import com.twitter.conversions.PercentOps._ + +object QigServiceClientModule + extends ThriftMethodBuilderClientModule[ + QigService.ServicePerEndpoint, + QigService.MethodPerEndpoint + ] + with MtlsClient { + override val label = "qig-service-srp" + override val dest = "/s/search-quality/qig-service" + + override protected def configureMethodBuilder( + injector: Injector, + methodBuilder: MethodBuilder + ): MethodBuilder = { + methodBuilder + .withTimeoutPerRequest(1000.milliseconds) + .withTimeoutTotal(1000.milliseconds) + .idempotent(1.percent) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/SimClustersAnnServiceClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/SimClustersAnnServiceClientModule.scala new file mode 100644 index 000000000..a1c566760 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/SimClustersAnnServiceClientModule.scala @@ -0,0 +1,71 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.google.inject.Provides +import com.twitter.conversions.PercentOps._ +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.config.TimeoutConfig +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.mtls.client.MtlsStackClient._ +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.inject.TwitterModule +import com.twitter.simclustersann.{thriftscala => t} +import javax.inject.Named +import javax.inject.Singleton + +object SimClustersAnnServiceClientModule extends TwitterModule { + @Provides + @Singleton + @Named(ModuleNames.ProdSimClustersANNServiceClientName) + def providesProdSimClustersANNServiceClient( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + timeoutConfig: TimeoutConfig, + statsReceiver: StatsReceiver, + ): t.SimClustersANNService.MethodPerEndpoint = { + val label = "simclusters-ann-server" + val dest = "/s/simclusters-ann/simclusters-ann" + + buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) + } + + @Provides + @Singleton + @Named(ModuleNames.SimClustersVideoANNServiceClientName) + def providesProdSimClustersVideoANNServiceClient( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + timeoutConfig: TimeoutConfig, + statsReceiver: StatsReceiver, + ): t.SimClustersANNService.MethodPerEndpoint = { + val label = "simclusters-ann-experimental-server" + val dest = "/s/simclusters-ann/simclusters-ann-experimental" + + buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) + } + + private def buildClient( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + timeoutConfig: TimeoutConfig, + statsReceiver: StatsReceiver, + dest: String, + label: String + ): t.SimClustersANNService.MethodPerEndpoint = { + val stats = statsReceiver.scope("clnt") + + val thriftClient = ThriftMux.client + .withMutualTls(serviceIdentifier) + .withClientId(clientId) + .withLabel(label) + .withStatsReceiver(stats) + .methodBuilder(dest) + .idempotent(5.percent) + .withTimeoutPerRequest(timeoutConfig.thriftSANNServiceClientTimeout) + .withRetryDisabled + .servicePerEndpoint[t.SimClustersANNService.ServicePerEndpoint] + + ThriftMux.Client.methodPerEndpoint(thriftClient) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/TweetyPieClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/TweetyPieClientModule.scala new file mode 100644 index 000000000..28f892c6e --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/TweetyPieClientModule.scala @@ -0,0 +1,56 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.mux.ClientDiscardedRequestException +import com.twitter.finagle.service.ReqRep +import com.twitter.finagle.service.ResponseClass +import com.twitter.finagle.service.RetryBudget +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient +import com.twitter.inject.Injector +import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule +import com.twitter.stitch.tweetypie.{TweetyPie => STweetyPie} +import com.twitter.tweet_mixer.config.TimeoutConfig +import com.twitter.tweetypie.thriftscala.TweetService +import com.twitter.util.Throw +import javax.inject.Singleton + +object TweetyPieClientModule + extends ThriftMethodBuilderClientModule[ + TweetService.ServicePerEndpoint, + TweetService.MethodPerEndpoint + ] + with MtlsClient { + + override val label = "tweetypie" + override val dest = "/s/tweetypie/tweetypie" + + override def retryBudget: RetryBudget = RetryBudget.Empty + + // We bump the success rate from the default of 0.8 to 0.9 since we're dropping the + // consecutive failures part of the default policy. + override def configureThriftMuxClient( + injector: Injector, + client: ThriftMux.Client + ): ThriftMux.Client = { + super + .configureThriftMuxClient(injector, client) + .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) + .withRequestTimeout(injector.instance[TimeoutConfig].thriftTweetypieClientTimeout) + .withSessionQualifier + .successRateFailureAccrual(successRate = 0.9, window = 30.seconds) + .withResponseClassifier { + case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable + } + } + + @Provides + @Singleton + def providesTweetyPie( + tweetyPieService: TweetService.MethodPerEndpoint + ): STweetyPie = { + STweetyPie(tweetyPieService) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/UserTweetGraphClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/UserTweetGraphClientModule.scala new file mode 100644 index 000000000..78c4de04b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/UserTweetGraphClientModule.scala @@ -0,0 +1,39 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.mux.ClientDiscardedRequestException +import com.twitter.finagle.service.ReqRep +import com.twitter.finagle.service.ResponseClass +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient +import com.twitter.inject.Injector +import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule +import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph +import com.twitter.util.Throw +import com.twitter.finagle.service.RetryBudget +import com.twitter.tweet_mixer.config.TimeoutConfig + +object UserTweetGraphClientModule + extends ThriftMethodBuilderClientModule[ + UserTweetGraph.ServicePerEndpoint, + UserTweetGraph.MethodPerEndpoint + ] + with MtlsClient { + + override val label = "user-tweet-graph" + override val dest = "/s/user-tweet-graph/user-tweet-graph" + + override def retryBudget: RetryBudget = RetryBudget.Empty + + override def configureThriftMuxClient( + injector: Injector, + client: ThriftMux.Client + ): ThriftMux.Client = + super + .configureThriftMuxClient(injector, client) + .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) + .withRequestTimeout(injector.instance[TimeoutConfig].thriftUserTweetGraphClientTimeout) + .withResponseClassifier { + case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/UserVideoGraphClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/UserVideoGraphClientModule.scala new file mode 100644 index 000000000..603fbfb8e --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/UserVideoGraphClientModule.scala @@ -0,0 +1,39 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.mux.ClientDiscardedRequestException +import com.twitter.finagle.service.ReqRep +import com.twitter.finagle.service.ResponseClass +import com.twitter.finagle.service.RetryBudget +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient +import com.twitter.inject.Injector +import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule +import com.twitter.recos.user_video_graph.thriftscala.UserVideoGraph +import com.twitter.tweet_mixer.config.TimeoutConfig +import com.twitter.util.Throw + +object UserVideoGraphClientModule + extends ThriftMethodBuilderClientModule[ + UserVideoGraph.ServicePerEndpoint, + UserVideoGraph.MethodPerEndpoint + ] + with MtlsClient { + + override val label = "user-video-graph" + override val dest = "/s/user-tweet-graph/user-video-graph" + + override def retryBudget: RetryBudget = RetryBudget.Empty + + override def configureThriftMuxClient( + injector: Injector, + client: ThriftMux.Client + ): ThriftMux.Client = + super + .configureThriftMuxClient(injector, client) + .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) + .withRequestTimeout(injector.instance[TimeoutConfig].thriftUserVideoGraphClientTimeout) + .withResponseClassifier { + case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/VecDBAnnServiceClientModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/VecDBAnnServiceClientModule.scala new file mode 100644 index 000000000..049762e28 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/module/thrift_client/VecDBAnnServiceClientModule.scala @@ -0,0 +1,56 @@ +package com.twitter.tweet_mixer.module.thrift_client + +import com.google.inject.Provides +import com.twitter.conversions.PercentOps._ +import com.twitter.tweet_mixer.model.ModuleNames +import com.twitter.tweet_mixer.config.TimeoutConfig +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.mtls.client.MtlsStackClient._ +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.inject.TwitterModule +import com.twitter.vecdb.{thriftscala => t} +import javax.inject.Named +import javax.inject.Singleton + +object VecDBAnnServiceClientModule extends TwitterModule { + @Provides + @Singleton + @Named(ModuleNames.VecDBAnnServiceClient) + def providesVecDBAnnServiceClient( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + timeoutConfig: TimeoutConfig, + statsReceiver: StatsReceiver, + ): t.VecDB.MethodPerEndpoint = { + val label = "vecdb-ann" + val dest = "/s/vecdb/frontend" + + buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) + } + + private def buildClient( + serviceIdentifier: ServiceIdentifier, + clientId: ClientId, + timeoutConfig: TimeoutConfig, + statsReceiver: StatsReceiver, + dest: String, + label: String + ): t.VecDB.MethodPerEndpoint = { + val stats = statsReceiver.scope("clnt") + + val thriftClient = ThriftMux.client + .withMutualTls(serviceIdentifier) + .withClientId(clientId) + .withLabel(label) + .withStatsReceiver(stats) + .methodBuilder(dest) + .idempotent(1.percent) + .withTimeoutPerRequest(timeoutConfig.candidateSourceTimeout) + .withRetryDisabled + .servicePerEndpoint[t.VecDB.ServicePerEndpoint] + + ThriftMux.Client.methodPerEndpoint(thriftClient) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/BUILD.bazel new file mode 100644 index 000000000..17a208d3f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/BUILD.bazel @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "src/scala/com/twitter/simclusters_v2/common", + "src/thrift/com/twitter/recos/signals:thrift-scala", + "src/thrift/com/twitter/recos/user_tweet_graph:user_tweet_graph-scala", + "src/thrift/com/twitter/recos/user_video_graph:user_video_graph-scala", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/decider", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CandidateSourceParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CandidateSourceParams.scala new file mode 100644 index 000000000..06a50726f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CandidateSourceParams.scala @@ -0,0 +1,30 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam + +object CandidateSourceParams { + val booleanFSOverrides = Seq(EventsEnabled, TrendsVideoEnabled) + + val boundedDoubleFsOverrides = Seq(EventsIrrelevanceDownrank) + + object EventsEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_events_enabled", + default = false + ) + + object EventsIrrelevanceDownrank + extends FSBoundedParam[Double]( + name = "tweet_mixer_events_irrelevance_downrank", + default = 0.5, + min = 0.0, + max = 10.0 + ) + + object TrendsVideoEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_trends_video_enabled", + default = false + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CertoTopicTweetsParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CertoTopicTweetsParams.scala new file mode 100644 index 000000000..47b9015e2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CertoTopicTweetsParams.scala @@ -0,0 +1,59 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam + +object CertoTopicTweetsParams { + + // Enables User Interested in Simclusters candidate source + object CertoTopicTweetsEnable + extends FSParam[Boolean]( + name = "certo_topic_tweets_enabled", + default = false + ) + + object UseProductContextTopicIds + extends FSParam[Boolean]( + name = "certo_topic_tweets_use_product_context_topic_ids", + default = false + ) + + object MaxNumCandidatesPerTopic + extends FSBoundedParam[Int]( + name = "certo_topic_tweets_max_num_candidates_per_topic", + min = 1, + max = 1000, + default = 100 + ) + + object MaxNumCandidates + extends FSBoundedParam[Int]( + name = "certo_topic_tweets_max_num_candidates", + min = 1, + max = 1000, + default = 400 + ) + + object MinCertoScore + extends FSBoundedParam[Double]( + name = "certo_topic_tweets_min_certo_score", + min = 0.0, + max = 1.0, + default = 0.005 + ) + + object MinFavCount + extends FSBoundedParam[Int]( + name = "certo_topic_tweets_min_fav_count", + min = 5, + max = 1000, + default = 10 + ) + + val booleanFSOverrides = Seq(CertoTopicTweetsEnable, UseProductContextTopicIds) + + val boundedIntFSOverrides = + Seq(MaxNumCandidatesPerTopic, MaxNumCandidates, MinFavCount) + + val boundedDoubleFsOverrides = Seq(MinCertoScore) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/ContentEmbeddingAnnParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/ContentEmbeddingAnnParams.scala new file mode 100644 index 000000000..762d27f8b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/ContentEmbeddingAnnParams.scala @@ -0,0 +1,57 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam +//UVG Related Params +object ContentEmbeddingAnnParams { + object MinScoreThreshold + extends FSBoundedParam[Double]( + name = "content_embedding_ann_tweets_min_score_threshold", + min = 0, + max = 1, + default = 0.6 + ) + + object MaxScoreThreshold + extends FSBoundedParam[Double]( + name = "content_embedding_ann_tweets_max_score_threshold", + min = 0, + max = 1, + default = 0.8 + ) + + object NumberOfCandidatesPerPost + extends FSBoundedParam[Int]( + name = "content_embedding_ann_tweets_num_candidates_per_post", + min = 0, + max = 50, + default = 5 + ) + + object DecayByCountry + extends FSParam[Boolean]( + name = "content_embedding_ann_tweets_decay_by_country", + default = true + ) + + object IncludeTextSource + extends FSParam[Boolean]( + name = "content_embedding_ann_tweets_include_text_source", + default = true + ) + + object IncludeMediaSource + extends FSParam[Boolean]( + name = "content_embedding_ann_tweets_include_media_source", + default = true + ) + + val boundedDoubleFSOverrides = + Seq(MaxScoreThreshold, MinScoreThreshold) + + val boundedIntFSOverrides = + Seq(NumberOfCandidatesPerPost) + + val booleanFSOverrides = + Seq(IncludeMediaSource, IncludeTextSource, DecayByCountry) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CuratedUserTlsPerLanguageParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CuratedUserTlsPerLanguageParams.scala new file mode 100644 index 000000000..3102d09b3 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/CuratedUserTlsPerLanguageParams.scala @@ -0,0 +1,21 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.timelines.configapi.FSParam + +object CuratedUserTlsPerLanguageParams { + object CuratedUserTlsPerLanguageTweetsEnable + extends FSParam[Boolean]( + name = "curated_user_tls_per_language_tweets_enable", + default = false + ) + + object CuratedUserTlsPerLanguageTweetsAuthorListParam + extends FSParam[Seq[Long]]( + name = "curated_user_tls_per_language_tweets_author_list_param", + Seq.empty + ) + + val booleanFSOverrides = Seq(CuratedUserTlsPerLanguageTweetsEnable) + + val longSeqFSOverrides = Seq(CuratedUserTlsPerLanguageTweetsAuthorListParam) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/EarlybirdInNetworkTweetsParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/EarlybirdInNetworkTweetsParams.scala new file mode 100644 index 000000000..53f5c8ea0 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/EarlybirdInNetworkTweetsParams.scala @@ -0,0 +1,13 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.timelines.configapi.FSParam + +object EarlybirdInNetworkTweetsParams { + + object EarlybirdInNetworkTweetsEnabled + extends FSParam[Boolean]( + name = "earlybird_in_network_tweets_enabled", + default = false + ) + val booleanFSOverrides = Seq(EarlybirdInNetworkTweetsEnabled) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/EvergreenParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/EvergreenParams.scala new file mode 100644 index 000000000..f2435f3e6 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/EvergreenParams.scala @@ -0,0 +1,17 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.timelines.configapi.FSBoundedParam + +object EvergreenParams { + + object TextSemanticBasedMaxResult + extends FSBoundedParam[Int]( + name = "evergreen_text_semantic_max_result", + default = 100, + min = 0, + max = 1000 + ) + + val boundedIntFSOverrides = + Seq(TextSemanticBasedMaxResult) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/GlobalParamConfigModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/GlobalParamConfigModule.scala new file mode 100644 index 000000000..19ded85d7 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/GlobalParamConfigModule.scala @@ -0,0 +1,10 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.inject.TwitterModule +import com.twitter.product_mixer.core.functional_component.configapi.registry.GlobalParamConfig + +object GlobalParamConfigModule extends TwitterModule { + override def configure(): Unit = { + bind[GlobalParamConfig].to[TweetMixerGlobalParamConfig] + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/HighQualitySourceSignalParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/HighQualitySourceSignalParams.scala new file mode 100644 index 000000000..c22dd35f4 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/HighQualitySourceSignalParams.scala @@ -0,0 +1,174 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.FSBoundedParam + +object HighQualitySourceSignalParams { + // Engagement Weights + object HighQualitySourceSignalBookmarkWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_bookmark_weight", + default = 5.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalFavWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_fav_weight", + default = 10.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalReplyWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_reply_weight", + default = 20.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalRetweetWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_retweet_weight", + default = 15.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalQuoteWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_quote_weight", + default = 15.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalShareWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_share_weight", + default = 5.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalVideoQualityViewWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_video_quality_view_weight", + default = 0.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalTweetDetailsClickWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_tweet_details_click_weight", + default = 0.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalTweetDetailsImpressionWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_tweet_details_impression_weight", + default = 0.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalNotInterestedWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_not_interested_weight", + default = -10.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalBlockWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_block_weight", + default = -20.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalMuteWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_mute_weight", + default = -15.0, + min = -100.0, + max = 100.0, + ) + + object HighQualitySourceSignalReportWeight + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_report_weight", + default = -20.0, + min = -100.0, + max = 100.0, + ) + + object EnableHighQualitySourceTweetV2 + extends FSParam[Boolean]( + name = "high_quality_source_signal_enable_high_quality_source_tweet", + default = false + ) + + object EnableHighQualitySourceUserV2 + extends FSParam[Boolean]( + name = "high_quality_source_signal_enable_high_quality_source_user", + default = false + ) + + object MaxHighQualitySourceSignalsV2 + extends FSBoundedParam[Int]( + name = "high_quality_source_signal_max_high_quality_source_signals", + default = 15, + min = 0, + max = 50 + ) + + // Decay Factor + object EnableTimeDecay + extends FSParam[Boolean]( + name = "high_quality_source_signal_enable_time_decay", + default = false + ) + + object TimeDecayRate + extends FSBoundedParam[Double]( + name = "high_quality_source_signal_time_decay_rate", + default = 4.0, + min = 0.0, + max = 540.0, + ) + + // Grouped for convenience + val boundedDoubleFSOverrides = Seq( + HighQualitySourceSignalBookmarkWeight, + HighQualitySourceSignalFavWeight, + HighQualitySourceSignalReplyWeight, + HighQualitySourceSignalRetweetWeight, + HighQualitySourceSignalQuoteWeight, + HighQualitySourceSignalShareWeight, + HighQualitySourceSignalVideoQualityViewWeight, + HighQualitySourceSignalTweetDetailsClickWeight, + HighQualitySourceSignalTweetDetailsImpressionWeight, + HighQualitySourceSignalNotInterestedWeight, + HighQualitySourceSignalBlockWeight, + HighQualitySourceSignalMuteWeight, + HighQualitySourceSignalReportWeight, + TimeDecayRate + ) + + val intFSOverrides = Seq( + MaxHighQualitySourceSignalsV2 + ) + + val booleanFSOverrides = Seq( + EnableTimeDecay, + EnableHighQualitySourceTweetV2, + EnableHighQualitySourceUserV2 + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopGrokTopicTweetsParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopGrokTopicTweetsParams.scala new file mode 100644 index 000000000..457b38b21 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopGrokTopicTweetsParams.scala @@ -0,0 +1,27 @@ +package com.twitter.tweet_mixer +package param + +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam + +object PopGrokTopicTweetsParams { + + // Enables Pop Grok Topic tweets candidate source + object PopGrokTopicTweetsEnable + extends FSParam[Boolean]( + name = "popular_grok_topic_tweets_enabled", + default = false + ) + + object MaxNumCandidates + extends FSBoundedParam[Int]( + name = "popular_grok_topic_tweets_max_num_candidates", + min = 1, + max = 2000, + default = 500 + ) + + val booleanFSOverrides = Seq(PopGrokTopicTweetsEnable) + + val boundedIntFSOverrides = Seq(MaxNumCandidates) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopularGeoTweetsParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopularGeoTweetsParams.scala new file mode 100644 index 000000000..10c11b725 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopularGeoTweetsParams.scala @@ -0,0 +1,48 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam + +object PopularGeoTweetsParams { + + // Enables Pop Geo candidate source + object PopularGeoTweetsEnable + extends FSParam[Boolean]( + name = "popular_geo_tweets_enabled", + default = false + ) + + object GeoSourceIds + extends FSParam[Seq[String]]( + name = "popular_geo_tweets_source_ids", + default = Seq( + "TOP_ER_RANK_BIRD_FILTER_VERIFIED", + "TOP_MEDIA_ER_RANK_BIRD_FILTER_VERIFIED", + "TOP_WER_VERIFIED", + "TOP_MEDIA_WER_VERIFIED" + ) + ) + + object MaxNumCandidatesPerTripSource + extends FSBoundedParam[Int]( + name = "popular_geo_tweets_max_num_candidates_per_source", + min = 1, + max = 1000, + default = 200 + ) + + object MaxNumPopGeoCandidates + extends FSBoundedParam[Int]( + name = "popular_geo_tweets_max_num_candidates", + min = 1, + max = 1000, + default = 400 + ) + + val booleanFSOverrides = Seq(PopularGeoTweetsEnable) + + val boundedIntFSOverrides = + Seq(MaxNumCandidatesPerTripSource, MaxNumPopGeoCandidates) + + val stringSeqFSOverrides = Seq(GeoSourceIds) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopularTopicTweetsParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopularTopicTweetsParams.scala new file mode 100644 index 000000000..6ffda5716 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/PopularTopicTweetsParams.scala @@ -0,0 +1,51 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSParam + +object PopularTopicTweetsParams { + + // Enables Popular Topics candidate source + object PopularTopicTweetsEnable + extends FSParam[Boolean]( + name = "popular_topic_tweets_enabled", + default = false + ) + + object SourceIds + extends FSParam[Seq[String]]( + name = "popular_topic_tweets_source_ids", + default = Seq("PopularTopicAll") + ) + + object MaxNumCandidatesPerTripSource + extends FSBoundedParam[Int]( + name = "popular_topic_tweets_max_num_candidates_per_source", + min = 1, + max = 1000, + default = 100 + ) + + object MaxNumCandidates + extends FSBoundedParam[Int]( + name = "popular_topic_tweets_max_num_candidates", + min = 1, + max = 1000, + default = 400 + ) + + object PopTopicIds + extends FSParam[Seq[Long]]( + name = "popular_topic_tweets_pop_topic_ids", + default = Seq.empty + ) + + val booleanFSOverrides = Seq(PopularTopicTweetsEnable) + + val boundedIntFSOverrides = + Seq(MaxNumCandidatesPerTripSource, MaxNumCandidates) + + val stringSeqFSOverrides = Seq(SourceIds) + + val longSeqFSOverrides = Seq(PopTopicIds) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/SimClustersAnnParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/SimClustersAnnParams.scala new file mode 100644 index 000000000..9c1ba6d8d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/SimClustersAnnParams.scala @@ -0,0 +1,204 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.simclusters_v2.common.ModelVersions +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSEnumParam +import com.twitter.timelines.configapi.FSParam + +object SimClustersANNParams { + + // Different SimClusters ANN cluster has its own config id (model slot) + object SimClustersANNConfigId + extends FSParam[String]( + name = "sann_simclusters_ann_config_id", + default = "Default" + ) + object SimClustersANNTweetBasedFavL2NormConfigId + extends FSParam[String]( + name = "sann_simclusters_ann_tweet_based_fav_l2_norm_config_id", + default = "FavL2Norm" + ) + + object SimClustersANNTweetBasedFavL2NormExplorationConfigId + extends FSParam[String]( + name = "sann_simclusters_ann_tweet_based_fav_l2_norm_exploration_config_id", + default = "FavL2NormExploration" + ) + + object SimClustersANNTweetBasedClusterDetailBasedFilteringConfigId + extends FSParam[String]( + name = "sann_simclusters_ann_tweet_based_cluster_detail_based_filtering_config_id", + default = "ClusterDetailBasedFiltering" + ) + + object SimClustersVideoANNConfigId + extends FSParam[String](name = "sann_video_ann_config_id", default = "Video") + + object ModelVersionParam + extends FSEnumParam[ModelVersions.Enum.type]( + name = "sann_simclusters_model_version_id", + default = ModelVersions.Enum.Model20M145K2020, + enum = ModelVersions.Enum + ) + + // SimClusters params for user interested in + object EnableProdSimClustersInterestedIn + extends FSParam[Boolean]( + name = "sann_enable_prod_simclusters_interested_in", + default = true + ) + + object EnableVideoSimClustersInterestedIn + extends FSParam[Boolean]( + name = "sann_enable_video_simclusters_interested_in", + default = false + ) + + // SimClusters params for producer based + object EnableProdSimClustersProducerBased + extends FSParam[Boolean]( + name = "sann_enable_prod_simclusters_producer_based", + default = true + ) + + object EnableVideoSimClustersProducerBased + extends FSParam[Boolean]( + name = "sann_enable_video_simclusters_producer_based", + default = false + ) + + // SimClusters params for tweet based + object EnableProdSimClustersTweetBased + extends FSParam[Boolean]( + name = "sann_enable_prod_simclusters_tweet_based", + default = true + ) + + object EnableProdSimClustersTweetBasedFavL2NormVersionBased + extends FSParam[Boolean]( + name = "sann_enable_prod_simclusters_tweet_based_fav_l2_norm_version_based", + default = false + ) + + object EnableProdSimClustersTweetBasedFavL2NormExplorationBased + extends FSParam[Boolean]( + name = "sann_enable_prod_simclusters_tweet_based_fav_l2_norm_exploration_based", + default = false + ) + object EnableProdSimClustersTweetBasedClusterDetailBasedFiltering + extends FSParam[Boolean]( + name = "sann_enable_prod_simclusters_tweet_based_cluster_detail_based_filtering", + default = false + ) + + object EnableVideoSimClustersTweetBased + extends FSParam[Boolean]( + name = "sann_enable_prod_video_simclusters_tweet_based", + default = false + ) + + // Min Score Params + object TweetBasedMinScoreParam + extends FSBoundedParam[Double]( + name = "sann_tweet_based_min_score", + default = 0.5, + min = 0.0, + max = 1.0 + ) + + object ProducerBasedMinScoreParam + extends FSBoundedParam[Double]( + name = "sann_producer_based_min_score", + default = 0.7, + min = 0.0, + max = 1.0 + ) + + object InterestedInMinScoreParam + extends FSBoundedParam[Double]( + name = "sann_interested_in_min_score", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object InterestedInMaxCandidatesParam + extends FSBoundedParam[Int]( + name = "sann_interested_in_max_candidates", + default = 200, + min = 0, + max = 10000 + ) + + object ProducerBasedMaxCandidatesParam + extends FSBoundedParam[Int]( + name = "sann_producer_based_max_candidates", + default = 200, + min = 0, + max = 10000 + ) + + object EnableProducerBasedMaxCandidatesParam + extends FSParam[Boolean]( + name = "sann_enable_producer_based_max_candidates", + default = false + ) + + object EnableSANNCacheParam + extends FSParam[Boolean]( + name = "sann_enable_cache", + default = true + ) + + object EnableAdditionalInterestedInEmbeddingTypesParam + extends FSParam[Boolean]( + name = "sann_enable_additional_interested_in_embedding_types", + default = false + ) + + val booleanFSOverrides = Seq( + EnableProdSimClustersInterestedIn, + EnableProdSimClustersProducerBased, + EnableProdSimClustersTweetBased, + EnableProdSimClustersTweetBasedFavL2NormVersionBased, + EnableProdSimClustersTweetBasedFavL2NormExplorationBased, + EnableProdSimClustersTweetBasedClusterDetailBasedFiltering, + EnableSANNCacheParam, + EnableVideoSimClustersTweetBased, + EnableVideoSimClustersProducerBased, + EnableVideoSimClustersInterestedIn, + EnableAdditionalInterestedInEmbeddingTypesParam, + EnableProducerBasedMaxCandidatesParam + ) + + val stringFSOverrides = Seq( + SimClustersANNConfigId, + SimClustersVideoANNConfigId, + SimClustersANNTweetBasedFavL2NormExplorationConfigId, + SimClustersANNTweetBasedClusterDetailBasedFilteringConfigId, + SimClustersANNTweetBasedFavL2NormConfigId + ) + + val enumFSOverrides = Seq(ModelVersionParam) + + val boundedDoubleFSOverrides = + Seq(TweetBasedMinScoreParam, ProducerBasedMinScoreParam, InterestedInMinScoreParam) + + val boundedIntFSOverrides = Seq(InterestedInMaxCandidatesParam, ProducerBasedMaxCandidatesParam) + + val InterestedInClusterParamMap = Map( + EnableProdSimClustersInterestedIn -> SimClustersANNConfigId, + EnableVideoSimClustersInterestedIn -> SimClustersVideoANNConfigId) + + val ProducerBasedClusterParamMap = Map( + EnableProdSimClustersProducerBased -> SimClustersANNConfigId, + EnableVideoSimClustersProducerBased -> SimClustersVideoANNConfigId) + + val TweetBasedClusterParamMap = Map( + EnableProdSimClustersTweetBased -> SimClustersANNConfigId, + EnableProdSimClustersTweetBasedFavL2NormVersionBased -> SimClustersANNTweetBasedFavL2NormConfigId, + EnableProdSimClustersTweetBasedFavL2NormExplorationBased -> SimClustersANNTweetBasedFavL2NormExplorationConfigId, + EnableProdSimClustersTweetBasedClusterDetailBasedFiltering -> SimClustersANNTweetBasedClusterDetailBasedFilteringConfigId, + EnableVideoSimClustersTweetBased -> SimClustersVideoANNConfigId + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/SkitTopicTweetsParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/SkitTopicTweetsParams.scala new file mode 100644 index 000000000..d3457ad14 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/SkitTopicTweetsParams.scala @@ -0,0 +1,93 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.HasDurationConversion +import com.twitter.timelines.configapi.DurationConversion +import com.twitter.util.Duration +import com.twitter.conversions.DurationOps._ +import com.twitter.timelines.configapi.FSParam + +object SkitTopicTweetsParams { + + // Enables User Interested in Simclusters candidate source + object SkitTopicTweetsEnable + extends FSParam[Boolean]( + name = "skit_topic_tweets_enabled", + default = false + ) + + object SkitHighPrecisionTopicTweetsEnable + extends FSParam[Boolean]( + name = "skit_topic_tweets_high_precisions_enabled", + default = false + ) + + object UseProductContextTopicIds + extends FSParam[Boolean]( + name = "skit_topic_tweets_use_product_context_topic_ids", + default = false + ) + + object MaxNumCandidatesPerTopic + extends FSBoundedParam[Int]( + name = "skit_topic_tweets_max_num_candidates_per_topic", + min = 1, + max = 1000, + default = 100 + ) + + object MaxNumCandidates + extends FSBoundedParam[Int]( + name = "skit_topic_tweets_max_num_candidates", + min = 1, + max = 1000, + default = 400 + ) + + object MinSkitScore + extends FSBoundedParam[Double]( + name = "skit_topic_tweets_min_certo_score", + min = 0.0, + max = 1.0, + default = 0.005 + ) + + object MinFavCount + extends FSBoundedParam[Int]( + name = "skit_topic_tweets_min_fav_count", + min = 5, + max = 1000, + default = 10 + ) + + object SemanticCoreVersionIdParam + extends FSBoundedParam[Long]( + name = "skit_topic_semantic_core_version_id", + default = , + max = Long.MaxValue, + min = 0L + ) + + object MaxTweetAge + extends FSBoundedParam[Duration]( + name = "topic_tweet_candidate_generation_max_tweet_age_hours", + default = 24.hours, + min = 12.hours, + max = 48.hours + ) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromHours + } + + val booleanFSOverrides = + Seq(SkitTopicTweetsEnable, UseProductContextTopicIds, SkitHighPrecisionTopicTweetsEnable) + + val boundedIntFSOverrides = + Seq(MaxNumCandidatesPerTopic, MaxNumCandidates, MinFavCount) + + val boundedDoubleFsOverrides = Seq(MinSkitScore) + + val boundedLongFSOverrides = Seq(SemanticCoreVersionIdParam) + + val boundedDurationFSOverrides = Seq(MaxTweetAge) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/TweetMixerGlobalParamConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/TweetMixerGlobalParamConfig.scala new file mode 100644 index 000000000..4271d25ac --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/TweetMixerGlobalParamConfig.scala @@ -0,0 +1,82 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.product_mixer.core.functional_component.configapi.registry.GlobalParamConfig +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Register Params that do not relate to a specific product. See GlobalParamConfig -> ParamConfig + * for hooks to register Params based on type. + */ +@Singleton +class TweetMixerGlobalParamConfig @Inject() () extends GlobalParamConfig { + + override val boundedIntFSOverrides = + TweetMixerGlobalParams.boundedIntFSOverrides ++ + USSParams.boundedIntFSOverrides ++ + HighQualitySourceSignalParams.intFSOverrides ++ + UTGParams.boundedIntFSOverrides ++ + UVGParams.boundedIntFSOverrides ++ + PopularGeoTweetsParams.boundedIntFSOverrides ++ + PopGrokTopicTweetsParams.boundedIntFSOverrides ++ + PopularTopicTweetsParams.boundedIntFSOverrides ++ + CertoTopicTweetsParams.boundedIntFSOverrides ++ + EvergreenParams.boundedIntFSOverrides ++ + SimClustersANNParams.boundedIntFSOverrides ++ + SkitTopicTweetsParams.boundedIntFSOverrides ++ + ContentEmbeddingAnnParams.boundedIntFSOverrides + + override val booleanDeciderOverrides = TweetMixerGlobalParams.booleanDeciderOverrides + + override val boundedDoubleDeciderOverrides = TweetMixerGlobalParams.boundedDoubleDeciderOverrides + + override val enumFSOverrides = + TweetMixerGlobalParams.enumFSOverrides ++ + USSParams.enumFSOverrides ++ + SimClustersANNParams.enumFSOverrides ++ + UTGParams.enumFSOverrides ++ + UVGParams.enumFSOverrides + + override val booleanFSOverrides = + TweetMixerGlobalParams.booleanFSOverrides ++ + USSParams.booleanFSOverrides ++ + HighQualitySourceSignalParams.booleanFSOverrides ++ + SimClustersANNParams.booleanFSOverrides ++ + UTGParams.booleanFSOverrides ++ + UVGParams.booleanFSOverrides ++ + PopularGeoTweetsParams.booleanFSOverrides ++ + PopGrokTopicTweetsParams.booleanFSOverrides ++ + PopularTopicTweetsParams.booleanFSOverrides ++ + CertoTopicTweetsParams.booleanFSOverrides ++ + EarlybirdInNetworkTweetsParams.booleanFSOverrides ++ + UserLocationParams.booleanFSOverrides ++ + ContentEmbeddingAnnParams.booleanFSOverrides ++ + CandidateSourceParams.booleanFSOverrides ++ + SkitTopicTweetsParams.booleanFSOverrides ++ + CuratedUserTlsPerLanguageParams.booleanFSOverrides + + override val stringFSOverrides = + SimClustersANNParams.stringFSOverrides ++ TweetMixerGlobalParams.stringFSOverrides + + override val boundedDurationFSOverrides = + TweetMixerGlobalParams.boundedDurationFSOverrides ++ USSParams.boundedDurationFSOverrides ++ SkitTopicTweetsParams.boundedDurationFSOverrides + + override val boundedDoubleFSOverrides = + TweetMixerGlobalParams.boundedDoubleFSOverrides ++ + HighQualitySourceSignalParams.boundedDoubleFSOverrides ++ + SimClustersANNParams.boundedDoubleFSOverrides ++ + UTGParams.boundedDoubleFSOverrides ++ + UVGParams.boundedDoubleFSOverrides ++ + CertoTopicTweetsParams.boundedDoubleFsOverrides ++ + ContentEmbeddingAnnParams.boundedDoubleFSOverrides + + override val stringSeqFSOverrides = + PopularGeoTweetsParams.stringSeqFSOverrides ++ + PopularTopicTweetsParams.stringSeqFSOverrides + + override val longSeqFSOverrides = + PopularTopicTweetsParams.longSeqFSOverrides ++ + CuratedUserTlsPerLanguageParams.longSeqFSOverrides + + override val boundedLongFSOverrides = SkitTopicTweetsParams.boundedLongFSOverrides +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/TweetMixerGlobalParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/TweetMixerGlobalParams.scala new file mode 100644 index 000000000..f02e08b4a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/TweetMixerGlobalParams.scala @@ -0,0 +1,1647 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.simclusters_v2.common.VersionId +import com.twitter.simclusters_v2.thriftscala.TwhinEmbeddingDataset +import com.twitter.timelines.configapi.DurationConversion +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSEnumParam +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.HasDurationConversion +import com.twitter.timelines.configapi.decider.BooleanDeciderParam +import com.twitter.timelines.configapi.decider.DeciderBoundedParam +import com.twitter.tweet_mixer.param.decider.DeciderKey +import com.twitter.util.Duration + +/** + * Instantiate Params that do not relate to a specific product. + * + * @see [[com.twitter.product_mixer.core.product.ProductParamConfig.supportedClientFSName]] + */ +object TweetMixerGlobalParams { + + object BlendingEnum extends Enumeration { + val RoundRobinBlending = Value + val SignalPriorityBlending = Value + val WeightedPriorityBlending = Value + val RoundRobinSourceBlending = Value + val RoundRobinSourceSignalBlending = Value + val RankingOnly = Value + } + + object MaxCandidateNumPerSourceKeyParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_candidate_per_sourcekey_max_num", + default = 200, + min = 0, + max = 1000 + ) + + object MaxTweetAgeHoursParam + extends FSBoundedParam[Duration]( + name = "tweet_mixer_max_tweet_age_hours", + default = 24.hours, + min = 1.hours, + max = 720.hours + ) + with HasDurationConversion { + + override val durationConversion: DurationConversion = DurationConversion.FromHours + } + + object MinVideoDurationParam + extends FSBoundedParam[Duration]( + name = "tweet_mixer_min_video_duration_seconds", + default = 6.seconds, + min = 1.seconds, + max = 720.seconds + ) + with HasDurationConversion { + + override val durationConversion: DurationConversion = DurationConversion.FromSeconds + } + + object ShortFormMinDurationParam + extends FSBoundedParam[Duration]( + name = "tweet_mixer_short_form_min_duration_seconds", + default = 5.seconds, + min = 1.seconds, + max = 14400.seconds // 4 hours + ) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromSeconds + } + + object LongFormMinDurationParam + extends FSBoundedParam[Duration]( + name = "tweet_mixer_long_form_min_duration_seconds", + default = 120.seconds, + min = 1.seconds, + max = 14400.seconds // 4 hours + ) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromSeconds + } + + object ShortFormMaxDurationParam + extends FSBoundedParam[Duration]( + name = "tweet_mixer_short_form_max_duration_seconds", + default = 300.seconds, + min = 1.seconds, + max = 14400.seconds // 4 hours + ) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromSeconds + } + + object LongFormMaxDurationParam + extends FSBoundedParam[Duration]( + name = "tweet_mixer_long_form_max_duration_seconds", + default = 14400.seconds, + min = 1.seconds, + max = 14400.seconds // 4 hours + ) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromSeconds + } + + object ShortFormMinAspectRatioParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_short_form_min_aspect_ratio", + default = 0.5625, + min = 0.0, + max = 4.0 + ) + + object LongFormMinAspectRatioParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_long_form_min_aspect_ratio", + default = 1.77, // 16:9 + min = 0.0, + max = 4.0 + ) + + object ShortFormMaxAspectRatioParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_short_form_max_aspect_ratio", + default = 1.0, + min = 0.0, + max = 4.0 + ) + + object LongFormMaxAspectRatioParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_long_form_max_aspect_ratio", + default = 2.0, // 16:9 + min = 0.0, + max = 4.0 + ) + + object ShortFormMinWidthParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_short_form_min_width", + default = 540, + min = 1, + max = 8192 + ) + + object LongFormMinWidthParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_long_form_min_width", + default = 1280, + min = 1, + max = 8192 + ) + + object ShortFormMinHeightParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_short_form_min_height", + default = 960, + min = 1, + max = 8192 + ) + + object LongFormMinHeightParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_long_form_min_height", + default = 720, + min = 1, + max = 8192 + ) + + object ShortFormMaxResultParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_short_form_max_result", + default = 200, + min = 1, + max = 2000 + ) + + object LongFormMaxResultParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_long_form_max_result", + default = 200, + min = 1, + max = 2000 + ) + + object MemeMaxResultParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_meme_max_result", + default = 200, + min = 1, + max = 2000 + ) + + object EnableDebugMode + extends FSParam[Boolean]( + name = "tweet_mixer_enable_debug_mode", + default = false + ) + + object EnableLowSignalUserCheck + extends FSParam[Boolean]( + name = "tweet_mixer_enable_low_signal_user_check", + default = false + ) + + object EnableMaxFollowersGate + extends FSParam[Boolean]( + name = "tweet_mixer_enable_max_followers_gate", + default = false + ) + + object MaxFollowersCountGateParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_max_followers_count_gate", + default = 100000, + min = 1, + max = 1000000000 + ) + + object LowSignalUserGrokTopicRatioParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_low_signal_user_grok_topic_ratio", + default = 0.7, + min = 0.0, + max = 1.0 + ) + + object LowSignalUserSimclustersRatioParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_low_signal_user_simclusters_ratio", + default = 0.15, + min = 0.0, + max = 1.0 + ) + + object LowSignalUserPopGeoRatioParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_low_signal_user_pop_geo_ratio", + default = 0.15, + min = 0.0, + max = 1.0 + ) + + object LowSignalUserBackfillRatioParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_low_signal_user_backfill_ratio", + default = 0.01, + min = 0.0, + max = 1.0 + ) + + object EnableNonEmptySearchHistoryUserCheck + extends FSParam[Boolean]( + name = "tweet_mixer_enable_non_empty_search_history_user_check", + default = false + ) + + object EnableTweetEntityServiceMigration + extends FSParam[Boolean]( + name = "tweet_mixer_enable_tweet_entity_service_migration", + default = false + ) + + object EnableTweetEntityServiceMigrationDiffy + extends FSParam[Boolean]( + name = "tweet_mixer_enable_tweet_entity_service_migration_diffy", + default = false + ) + + object EnableUSSFeatureHydrator + extends FSParam[Boolean]( + name = "tweet_mixer_enable_uss_feature_hydrator", + default = true + ) + + object EnableUSSGrokCategoryFeatureHydrator + extends FSParam[Boolean]( + name = "tweet_mixer_enable_uss_grok_category_feature_hydrator", + default = false + ) + + object EnableUSSDeepRetrievalTweetEmbeddingFeatureHydrator + extends FSParam[Boolean]( + name = "tweet_mixer_enable_uss_deep_retrieval_tweet_embedding_feature_hydrator", + default = false + ) + + object EnableImpressionBloomFilterHydrator + extends FSParam[Boolean]( + name = "tweet_mixer_enable_impression_bloom_filter_hydrator", + default = false + ) + + object EnableGizmoduckQueryFeatureHydrator + extends FSParam[Boolean]( + name = "tweet_mixer_enable_gizmoduck_hydrator", + default = false + ) + + object EnableVideoBloomFilterHydrator + extends FSParam[Boolean]( + name = "tweet_mixer_enable_video_bloom_filter_hydrator", + default = false + ) + + object EnableSignalInfoHydrator + extends FSParam[Boolean]( + name = "tweet_mixer_enable_signal_info_hydrator", + default = true + ) + + object EnableRequestCountryPlaceIdHydrator + extends FSParam[Boolean]( + name = "tweet_mixer_enable_request_country_place_id_hydrator", + default = true + ) + + object EnableUserTopicIdsHydrator + extends FSParam[Boolean]( + name = "tweet_mixer_enable_user_topic_ids_hydrator", + default = false + ) + + object EnableGrokFilter + extends FSParam[Boolean]( + name = "tweet_mixer_enable_grok_filter", + default = false + ) + + object EnableVideoTweetFilter + extends FSParam[Boolean]( + name = "tweet_mixer_enable_video_tweet_filter", + default = false + ) + + object EnableShortVideoFilter + extends FSParam[Boolean]( + name = "tweet_mixer_enable_short_video_filter", + default = false + ) + + object EnableLongFormVideoFilter + extends FSParam[Boolean]( + name = "tweet_mixer_enable_long_form_video_filter", + default = false + ) + + object EnableShortFormVideoFilter + extends FSParam[Boolean]( + name = "tweet_mixer_enable_short_form_video_filter", + default = false + ) + + object EnablePortraitVideoFilter + extends FSParam[Boolean]( + name = "tweet_mixer_enable_portrait_video_filter", + default = false + ) + + object EnableMediaIdDedupFilter + extends FSParam[Boolean]( + name = "tweet_mixer_enable_media_id_dedup_filter", + default = false + ) + + object EnableMediaMetadataCandidateFeatureHydrator + extends FSParam[Boolean]( + name = "tweet_mixer_enable_media_metadata_candidate_feature_hydrator", + default = false + ) + + object EnableMediaClusterIdDedupFilter + extends FSParam[Boolean]( + name = "tweet_mixer_enable_media_cluster_id_dedup_filter", + default = false + ) + + object EnableHydraScoringSideEffect + extends BooleanDeciderParam(decider = DeciderKey.EnableHydraScoringSideEffect) + + object EnableEvergreenVideosSideEffect + extends BooleanDeciderParam(decider = DeciderKey.EnableEvergreenVideosSideEffect) + + object BlendingParam + extends FSEnumParam[BlendingEnum.type]( + name = "tweet_mixer_blending_id", + default = BlendingEnum.RoundRobinBlending, + enum = BlendingEnum + ) + + object HaploliteTweetsBasedEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_haplolite_based_enabled", + default = false + ) + + object MaxHaploliteTweetsParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_max_haplolite_tweets", + default = 4500, + min = 0, + max = 10000 + ) + + object TwhinConsumerBasedEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_twhin_consumer_based_enabled", + default = true + ) + + object TwhinRebuildTweetSimilarityDatasetEnum extends Enumeration { + private def normalizeDatasetName(originalName: String): String = { + originalName.replace("_", "-").toLowerCase + } + + val TwhinTweet: Value = Value( + normalizeDatasetName(TwhinEmbeddingDataset.TwhinTweet.originalName)) + val RefreshedTwhinTweet: Value = Value( + normalizeDatasetName(TwhinEmbeddingDataset.RefreshedTwhinTweet.originalName)) + val enumToTwhinRebuildVersionIdMap: Map[ + TwhinRebuildTweetSimilarityDatasetEnum.Value, + VersionId + ] = Map( + TwhinTweet -> TwhinEmbeddingDataset.TwhinTweet.value, + RefreshedTwhinTweet -> TwhinEmbeddingDataset.RefreshedTwhinTweet.value + ) + } + + // Which VecDB dataset to use in the tweet based TwHIN rebuild ANN candidate source + object TwhinRebuildTweetSimilarityDatasetParam + extends FSEnumParam[TwhinRebuildTweetSimilarityDatasetEnum.type]( + name = "tweet_mixer_twhin_rebuild_tweet_similarity_dataset_id", + default = TwhinRebuildTweetSimilarityDatasetEnum.TwhinTweet, + enum = TwhinRebuildTweetSimilarityDatasetEnum + ) + + // Enables Tweet Based TwHIN rebuild ANN candidate source + object TwhinRebuildTweetSimilarityEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_twhin_rebuild_tweet_similarity_enabled", + default = false + ) + + // Enables User Tweet Based TwHIN rebuild ANN candidate source + object TwhinRebuildUserTweetSimilarityEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_twhin_rebuild_user_tweet_similarity_enabled", + default = false + ) + + object TwhinRebuildUserTweetSimilarityMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_twhin_rebuild_user_tweet_similarity_max_candidates", + default = 100, + min = 0, + max = 2000 + ) + + object TwhinRebuildUserTweetVectorDBName + extends FSParam[String]( + name = "tweet_mixer_twhin_rebuild_user_tweet_vectordb_name", + default = "refreshed-twhin-tweet" + ) + + // How many candidates to retrieve for each tweet signal in TwHIN rebuild ANN candidate source + object TwhinRebuildTweetSimilarityMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_twhin_rebuild_tweet_similarity_max_candidates", + default = 100, + min = 0, + max = 200 + ) + + object TwhinTweetSimilarityEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_twhin_tweet_similarity_enabled", + default = false + ) + + object SimclustersInterestedInEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_simclusters_interested_in_enabled", + default = true + ) + + object SimclustersProducerBasedEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_simclusters_producer_based_enabled", + default = true + ) + + object SimclustersTweetBasedEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_simclusters_tweet_based_enabled", + default = true + ) + + object SimclustersPromotedCreatorEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_simclusters_promoted_creator_enabled", + default = false + ) + + object UTEGEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_uteg_enabled", + default = false + ) + + object UTGTweetBasedEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_utg_tweet_based_enabled", + default = true + ) + + object UTGProducerBasedEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_utg_producer_based_enabled", + default = true + ) + + object UTGTweetBasedExpansionEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_utg_tweet_based_expansion_enabled", + default = true + ) + + object UVGTweetBasedEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_uvg_tweet_based_enabled", + default = true + ) + + object UVGTweetBasedExpansionEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_uvg_tweet_based_expansion_enabled", + default = true + ) + + object ContentExplorationEmbeddingSimilarityEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_embedding_similarity_enabled", + default = false + ) + + object ContentExplorationMediaEmbeddingSimilarityEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_media_embedding_similarity_enabled", + default = false + ) + + object ContentExplorationMultimodalEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_multimodal_enabled", + default = false + ) + + object ContentExplorationNonVideoTweetFeaturesEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_non_video_tweet_features_enabled", + default = false + ) + + object ContentExplorationVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_content_exploration_vectordb_collection_name", + default = "content-exploration-text-emb" + ) + + object ContentExplorationEmbeddingANNMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_embedding_ann_max_candidates", + default = 10, + min = 0, + max = 100 + ) + + object ContentExplorationEmbeddingANNScoreThreshold + extends FSBoundedParam[Double]( + name = "tweet_mixer_content_exploration_embedding_ann_score_threshold", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object UserInterestSummarySimilarityEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_user_interest_summary_similarity_enabled", + default = false + ) + + object UserInterestSummarySimilarityVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_user_interest_summary_similarity_vectordb_collection_name", + default = "content-exploration-description-text-emb" + ) + + object UserInterestSummarySimilarityMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_user_interest_summary_similarity_max_candidates", + default = 10, + min = 0, + max = 100 + ) + + object UserInterestSummarySimilarityScoreThreshold + extends FSBoundedParam[Double]( + name = "tweet_mixer_user_interest_summary_similarity_score_threshold", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object ContentExplorationEmbeddingSimilarityTierTwoEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_embedding_similarity_tier_two_enabled", + default = false + ) + + object ContentExplorationEmbeddingSimilarityTierTwoVectorDBCollectionName + extends FSParam[String]( + name = + "tweet_mixer_content_exploration_embedding_similarity_tier_two_vectordb_collection_name", + default = "content-exploration-description-text-emb" + ) + + object ContentExplorationEmbeddingSimilarityTierTwoMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_embedding_similarity_tier_two_max_candidates", + default = 10, + min = 0, + max = 100 + ) + + object ContentExplorationEmbeddingSimilarityTierTwoScoreThreshold + extends FSBoundedParam[Double]( + name = "tweet_mixer_content_exploration_embedding_similarity_tier_two_score_threshold", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object ContentExplorationDRTweetTweetEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_dr_tweet_tweet_enabled", + default = false + ) + + object ContentExplorationDRTweetTweetVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_content_exploration_dr_tweet_tweet_vectordb_collection_name", + default = "content-exploration-deep-retrieval" + ) + + object ContentExplorationDRTweetTweetMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_dr_tweet_tweet_max_candidates", + default = 10, + min = 0, + max = 100 + ) + + object ContentExplorationDRTweetTweetScoreThreshold + extends FSBoundedParam[Double]( + name = "tweet_mixer_content_exploration_dr_tweet_tweet_score_threshold", + default = -1.0, + min = -1.0, + max = 1.0 + ) + + object ContentExplorationDRTweetTweetEnableRandomEmbedding + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_dr_tweet_tweet_enable_random_embedding", + default = false + ) + + object ContentExplorationEvergreenDRTweetTweetEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_evergreen_dr_tweet_tweet_enabled", + default = false + ) + + object ContentExplorationEvergreenDRTweetTweetVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_content_exploration_evergreen_dr_tweet_tweet_vectordb_collection_name", + default = "content-exploration-deep-retrieval-exp1" + ) + + object ContentExplorationEvergreenDRTweetTweetMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_evergreen_dr_tweet_tweet_max_candidates", + default = 10, + min = 0, + max = 100 + ) + + object ContentExplorationEvergreenDRTweetTweetScoreThreshold + extends FSBoundedParam[Double]( + name = "tweet_mixer_content_exploration_evergreen_dr_tweet_tweet_score_threshold", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object ContentExplorationDRTweetTweetTierTwoEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_dr_tweet_tweet_tier_two_enabled", + default = false + ) + + object ContentExplorationDRTweetTweetTierTwoVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_content_exploration_dr_tweet_tweet_tier_two_vectordb_collection_name", + default = "content-exploration-deep-retrieval" + ) + + object ContentExplorationDRTweetTweetTierTwoMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_dr_tweet_tweet_tier_two_max_candidates", + default = 10, + min = 0, + max = 100 + ) + + object ContentExplorationDRTweetTweetTierTwoScoreThreshold + extends FSBoundedParam[Double]( + name = "tweet_mixer_content_exploration_dr_tweet_tweet_tier_two_score_threshold", + default = -1.0, + min = -1.0, + max = 1.0 + ) + + object ContentExplorationDRUserTweetEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_dr_user_tweet_enabled", + default = false + ) + + object ContentExplorationDRUserEmbeddingModelName + extends FSParam[String]( + name = "tweet_mixer_content_exploration_dr_user_embedding_model_name", + default = "deep_retrieval_exp2" + ) + + object ContentExplorationDRUserTweetVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_content_exploration_dr_user_tweet_vectordb_collection_name", + default = "content-exploration-deep-retrieval" + ) + + object ContentExplorationDRUserTweetMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_dr_user_tweet_max_candidates", + default = 50, + min = 0, + max = 100 + ) + + object ContentExplorationDRUserTweetTierTwoEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_dr_user_tweet_tier_two_enabled", + default = false + ) + + object ContentExplorationDRUserTweetTierTwoMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_dr_user_tweet_tier_two_max_candidates", + default = 50, + min = 0, + max = 100 + ) + + object ContentExplorationMaxViewCountThreshold + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_max_view_count_threshold", + default = 100, + min = 0, + max = 1000000000 + ) + + object ContentExplorationTier2MaxViewCountThreshold + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_tier2_max_view_count_threshold", + default = 900, + min = 0, + max = 1000000000 + ) + + object ContentExplorationOnceInTimesShow + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_once_in_times_show", + default = 3, + min = 1, + max = 20 + ) + + object ContentExplorationMinHoursSinceLastRequestParam + extends FSBoundedParam[Duration]( + name = "tweet_mixer_content_exploration_min_time_since_last_request_in_hours", + default = 2.hours, + min = 0.hours, + max = 24.hours + ) + with HasDurationConversion { + + override val durationConversion: DurationConversion = DurationConversion.FromHours + } + + object ContentExplorationSimclusterColdPostsEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_content_exploration_simcluster_cold_posts_enabled", + default = false + ) + + object ContentExplorationSimclusterColdPostsMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_simcluster_cold_posts_max_candidates", + default = 100, + min = 0, + max = 1000 + ) + + object ContentExplorationSimclusterColdPostsPostsPerSimcluster + extends FSBoundedParam[Int]( + name = "tweet_mixer_content_exploration_simcluster_cold_posts_posts_per_simcluster", + default = 20, + min = 0, + max = 100 + ) + + object LastNonPollingTimeFeatureHydratorEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_last_non_polling_time_feature_hydrator_enabled", + default = false + ) + + object EvergreenDRUserTweetEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_evergreen_dr_user_tweet_enabled", + default = false + ) + + object EvergreenDRUserEmbeddingModelName + extends FSParam[String]( + name = "tweet_mixer_evergreen_dr_user_embedding_model_name", + default = "deep_retrieval_exp3" + ) + + object EvergreenDRUserTweetVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_evergreen_dr_user_tweet_vectordb_collection_name", + default = "content-exploration-deep-retrieval-exp1" + ) + + object EvergreenDRUserTweetMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_evergreen_dr_user_tweet_max_candidates", + default = 50, + min = 0, + max = 1000 + ) + + object EvergreenDRCrossBorderUserTweetEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_evergreen_dr_cross_border_user_tweet_enabled", + default = false + ) + + object EvergreenDRCrossBorderUserEmbeddingModelName + extends FSParam[String]( + name = "tweet_mixer_evergreen_dr_cross_border_user_embedding_model_name", + default = "deep_retrieval_exp3" + ) + + object EvergreenDRCrossBorderUserTweetVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_evergreen_dr_cross_border_user_tweet_vectordb_collection_name", + default = "content-exploration-cross-border-dr-exp1" + ) + + object EvergreenDRCrossBorderUserTweetMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_evergreen_dr_cross_border_user_tweet_max_candidates", + default = 50, + min = 0, + max = 1000 + ) + + // Enables User Based Deep Retrieval ANN candidate source + object DeepRetrievalUserTweetANNEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_deep_retrieval_user_tweet_ann_enabled", + default = false + ) + + object DeepRetrievalCategoricalUserTweetANNEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_deep_retrieval_categorical_user_tweet_ann_enabled", + default = false + ) + + object DeepRetrievalIsHighQualityEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_deep_retrieval_is_high_quality_enabled", + default = false + ) + + object DeepRetrievalIsLowNegEngRatioEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_deep_retrieval_is_low_neg_ratio_enabled", + default = false + ) + + object DeepRetrievalTweetTweetANNEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_ann_enabled", + default = false + ) + + object DeepRetrievalTweetTweetEmbeddingANNEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_embedding_ann_enabled", + default = false + ) + + object DeepRetrievalTweetTweetEmbeddingRandomSize + extends FSBoundedParam[Int]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_embedding_random_size", + default = 7, + min = 0, + max = 100000 + ) + + object DeepRetrievalTweetTweetEmbeddingTimeDecay + extends FSBoundedParam[Double]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_embedding_time_decay", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object DeepRetrievalTweetTweetEmbeddingSeedMaxAgeInDays + extends FSBoundedParam[Duration]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_embedding_seed_max_age_in_days", + default = 1.days, + min = 0.days, + max = 2000.days + ) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromDays + } + + object DeepRetrievalTweetTweetEmbeddingTimeOfDayBoostWeight + extends FSBoundedParam[Double]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_embedding_time_of_day_boost_weight", + default = 1.0, + min = 0.0, + max = 1000.0 + ) + + object DeepRetrievalTweetTweetEmbeddingDayOfWeekBoostWeight + extends FSBoundedParam[Double]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_embedding_day_of_week_boost_weight", + default = 1.0, + min = 0.0, + max = 1000.0 + ) + + object DeepRetrievalTweetTweetEmbeddingEngagementBoostWeight + extends FSBoundedParam[Double]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_embedding_engagement_boost_weight", + default = 1.0, + min = 0.0, + max = 1000.0 + ) + + object DeepRetrievalTweetTweetEmbeddingMinPriority + extends FSBoundedParam[Int]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_embedding_min_priority", + default = 3, + min = -1, + max = 5 + ) + + object DeepRetrievalVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_deep_retrieval_vectordb_collection_name", + default = "tweet-deep-retrieval" + ) + + object DeepRetrievalI2iVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_deep_retrieval_i2i_vectordb_collection_name", + default = "tweet-deep-retrieval-exp3" + ) + + object DeepRetrievalI2iEmbVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_deep_retrieval_i2i_emb_vectordb_collection_name", + default = "content-exploration-deep-retrieval" + ) + + object DeepRetrievalEnableGPU + extends FSParam[Boolean]( + name = "tweet_mixer_deep_retrieval_enable_gpu", + default = false + ) + + object DeepRetrievalUserTweetANNMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_deep_retrieval_user_tweet_ann_max_candidates", + default = 200, + min = 0, + max = 5000 + ) + + object DeepRetrievalNonVideoTweetFeaturesEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_deep_retrieval_non_video_tweet_features_enabled", + default = false + ) + + object DeepRetrievalFilterOldSignalsEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_deep_retrieval_filter_old_signals_enabled", + default = false + ) + + object DeepRetrievalTweetTweetANNMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_ann_max_candidates", + default = 50, + min = 0, + max = 500 + ) + + object DeepRetrievalTweetTweetANNScoreThreshold + extends FSBoundedParam[Double]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_ann_score_threshold_param", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object USSDeepRetrievalSimilarityTweetTweetANNScoreThreshold + extends FSBoundedParam[Double]( + name = "tweet_mixer_uss_deep_retrieval_similarity_embedding_threshold_param", + default = 0.8, + min = 0.0, + max = 1.0 + ) + + object USSDeepRetrievalStratoTimeout + extends FSBoundedParam[Int]( + name = "tweet_mixer_uss_deep_retrieval_strato_timeout_param", + default = 130, + min = 100, + max = 500 + ) + + object OutlierDeepRetrievalStratoTimeout + extends FSBoundedParam[Int]( + name = "tweet_mixer_outlier_deep_retrieval_strato_timeout_param", + default = 200, + min = 100, + max = 500 + ) + + object DeepRetrievalUserTweetANNScoreThreshold + extends FSBoundedParam[Double]( + name = "tweet_mixer_deep_retrieval_user_tweet_ann_score_threshold_param", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object DeepRetrievalTweetTweetEmbeddingANNScoreThreshold + extends FSBoundedParam[Double]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_embedding_ann_score_threshold_param", + default = 0.5, + min = 0.0, + max = 1.0 + ) + + object DeepRetrievalTweetTweetANNScoreMaxThreshold + extends FSBoundedParam[Double]( + name = "tweet_mixer_deep_retrieval_tweet_embedding_score_max_threshold_param", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + object DeepRetrievalTweetTweetEmbeddingANNMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_deep_retrieval_tweet_tweet_embedding_ann_max_candidates", + default = 20, + min = 0, + max = 500 + ) + + object MediaDeepRetrievalUserTweetANNEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_media_deep_retrieval_user_tweet_ann_enabled", + default = false + ) + + object MediaDeepRetrievalIsHighQualityEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_media_deep_retrieval_is_high_quality_enabled", + default = false + ) + + object MediaDeepRetrievalIsLowNegEngRatioEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_media_deep_retrieval_is_low_neg_ratio_enabled", + default = false + ) + + object MediaEvergreenDeepRetrievalUserTweetANNEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_media_evergreen_deep_retrieval_user_tweet_ann_enabled", + default = false + ) + + object MediaPromotedCreatorDeepRetrievalUserTweetANNEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_media_promoted_creator_deep_retrieval_user_tweet_ann_enabled", + default = false + ) + + object MediaDeepRetrievalVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_media_deep_retrieval_vectordb_collection_name", + default = "tweet-deep-retrieval-media" + ) + + object MediaEvergreenDeepRetrievalVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_media_evergreen_deep_retrieval_vectordb_collection_name", + default = "tweet-deep-retrieval-media-exp2" + ) + + object MediaPromotedCreatorDeepRetrievalVectorDBCollectionName + extends FSParam[String]( + name = "tweet_mixer_media_promoted_creator_deep_retrieval_vectordb_collection_name", + default = "creator-incentive-retrieval-v1" + ) + + object MediaDeepRetrievalUserTweetANNMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_media_deep_retrieval_user_tweet_ann_max_candidates", + default = 200, + min = 0, + max = 1000 + ) + + object MediaEvergreenDeepRetrievalUserTweetANNMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_media_evergreen_deep_retrieval_user_tweet_ann_max_candidates", + default = 200, + min = 0, + max = 1000 + ) + + object MediaPromotedCreatorDeepRetrievalUserTweetANNMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_media_promoted_creator_deep_retrieval_user_tweet_ann_max_candidates", + default = 200, + min = 0, + max = 1000 + ) + + object EnableRelatedCreatorParam + extends FSParam[Boolean]( + name = "tweet_mixer_related_creator_enabled", + default = false + ) + + object MediaRelatedCreatorMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_media_related_creator_max_candidates", + default = 100, + min = 0, + max = 200 + ) + + object EnableDeepRetrievalAdhocDecider + extends BooleanDeciderParam( + DeciderKey.EnableDeepRetrievalAdhocSideEffect + ) + + object DeepRetreivalAdhocMaxResultsDivByTenDecider + extends DeciderBoundedParam[Double]( + DeciderKey.DeepRetrievalSideEffectNumCandidatesDivByTen, + 100.0, + 0.1, + 10000.0) + + object EnableTESMigrationDiffy + extends BooleanDeciderParam( + DeciderKey.EnableTESMigrationDiffy + ) + + object ControlAiEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_control_ai_enabled", + default = false + ) + + object ControlAiTopicEmbeddingANNMaxCandidates + extends FSBoundedParam[Int]( + name = "tweet_mixer_control_ai_topic_embedding_ann_max_candidates", + default = 300, + min = 0, + max = 1000 + ) + + // Enables Hydra Scoring Pipeline + object HydraScoringPipelineEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_hydra_scoring_pipeline_enabled", + default = false + ) + + // Enables Hydra Based Sorting + object HydraBasedSortingEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_hydra_based_sorting_enabled", + default = false + ) + + //Hydra Model Nmae + object HydraModelName + extends FSParam[String]( + name = "tweet_mixer_hydra_model_name", + default = "model1" + ) + + object DeepRetrievalModelName + extends FSParam[String]( + name = "tweet_mixer_deep_retrieval_model_name", + default = "deep_retrieval" + ) + + object DeepRetrievalI2iEmbModelName + extends FSParam[String]( + name = "tweet_mixer_deep_retrieval_i2i_embmodel_name", + default = "deep_retrieval_exp2" + ) + + object USSDeepRetrievalI2iEmbModelName + extends FSParam[String]( + name = "tweet_mixer_uss_deep_retrieval_i2i_embmodel_name", + default = "deep_retrieval_exp4" + ) + + object OutlierTweetEmbeddingModelNameParam + extends FSParam[String]( + name = "tweet_mixer_outlier_tweet_embedding_model_name", + default = "deep_retrieval_exp3" + ) + + object DeepRetrievalAddUserEmbeddingGaussianNoise + extends FSParam[Boolean]( + name = "tweet_mixer_deep_retrieval_add_user_embedding_gaussian_noise", + default = false + ) + + object DeepRetrievalUserEmbeddingGaussianNoiseParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_deep_retrieval_user_embedding_gaussian_noise_param", + default = 1.0, + min = 0.0, + max = 100.0 + ) + + object MediaDeepRetrievalModelName + extends FSParam[String]( + name = "tweet_mixer_media_deep_retrieval_model_name", + default = "deep_retrieval_media" + ) + + object MediaEvergreenDeepRetrievalModelName + extends FSParam[String]( + name = "tweet_mixer_media_evergreen_deep_retrieval_model_name", + default = "evergreen_retrieval_v1" + ) + + object MediaPromotedCreatorDeepRetrievalModelName + extends FSParam[String]( + name = "tweet_mixer_media_promoted_creator_deep_retrieval_model_name", + default = "creator_incentive_retrieval_v1" + ) + + object VideoScoreWeightParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_video_score_weight", + default = 0.6, + min = -1, + max = 100 + ) + + object ReplyScoreWeightParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_reply_score_weight", + default = 0.0, + min = -1, + max = 100 + ) + + object ControlAiShowMoreWeightParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_control_ai_show_more_weight", + default = 0.0, + min = -1000, + max = 1000 + ) + + object ControlAiShowLessWeightParam + extends FSBoundedParam[Double]( + name = "tweet_mixer_control_ai_show_less_weight", + default = 0.0, + min = -1000, + max = 1000 + ) + + object ExperimentBucketIdentifierParam + extends FSParam[String]( + name = "tweet_mixer_experiment_bucket_identifier", + default = "Interleave" + ) + + object EvergreenVideosEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_evergreen_videos_enabled", + default = false + ) + + object GrokFilterFeatureHydratorEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_grok_filter_feature_hydrator_enabled", + default = false + ) + + object ViewCountInfoOnTweetFeatureHydratorEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_view_count_info_on_tweet_feature_hydrator_enabled", + default = false + ) + + object SemanticVideoCandidatePipelineEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_semantic_video_candidate_pipeline_enabled", + default = true + ) + + object TwitterClipV0LongVideoCandidatePipelineEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_twitter_clip_v0_long_video_candidate_pipeline_enabled", + default = false + ) + + object TwitterClipV0ShortVideoCandidatePipelineEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_twitter_clip_v0_short_video_candidate_pipeline_enabled", + default = false + ) + + object EvergreenVideosPaginationNumParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_evergreen_videos_pagination_num", + default = 200, + min = 0, + max = 2000 + ) + + object EvergreenVideosMaxFollowUsersParam + extends FSBoundedParam[Int]( + name = "tweet_mixer_evergreen_videos_max_follow_users", + default = 500, + min = 0, + max = 2000 + ) + + object QigSearchHistoryTweetsEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_qig_search_history_tweets_enabled", + default = false + ) + + object QigSearchHistoryCandidateSourceEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_qig_search_history_candidate_source_enabled", + default = false + ) + + object QigSearchHistoryTweetsEnableLanguageFilter + extends FSParam[Boolean]( + name = "tweet_mixer_qig_search_history_tweets_enable_language_filter", + default = false + ) + + object QigSearchHistoryTopKLimit + extends FSBoundedParam[Int]( + name = "tweet_mixer_qig_search_history_top_k_limit", + default = 500, + min = 0, + max = 10000 + ) + + object ScribeRetrievedCandidatesParam + extends FSParam[Boolean]( + name = "tweet_mixer_scribe_retrieved_candidates", + default = false + ) + + object EnableContentEmbeddingAnnTweets + extends FSParam[Boolean]( + name = "tweet_mixer_enable_content_embedding_tweet_based", + default = false + ) + + object UVGHighQualityTweetBasedExpansionEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_uvg_high_quality_tweet_based_expansion_enabled", + default = false + ) + + val boundedDurationFSOverrides = Seq( + MaxTweetAgeHoursParam, + MinVideoDurationParam, + ShortFormMinDurationParam, + ShortFormMaxDurationParam, + LongFormMinDurationParam, + LongFormMaxDurationParam, + ContentExplorationMinHoursSinceLastRequestParam, + DeepRetrievalTweetTweetEmbeddingSeedMaxAgeInDays, + ) + + val boundedIntFSOverrides = Seq( + ControlAiTopicEmbeddingANNMaxCandidates, + ContentExplorationDRTweetTweetMaxCandidates, + ContentExplorationEvergreenDRTweetTweetMaxCandidates, + ContentExplorationDRTweetTweetTierTwoMaxCandidates, + ContentExplorationEmbeddingANNMaxCandidates, + UserInterestSummarySimilarityMaxCandidates, + ContentExplorationEmbeddingSimilarityTierTwoMaxCandidates, + ContentExplorationMaxViewCountThreshold, + ContentExplorationTier2MaxViewCountThreshold, + ContentExplorationDRUserTweetMaxCandidates, + ContentExplorationDRUserTweetTierTwoMaxCandidates, + ContentExplorationOnceInTimesShow, + ContentExplorationSimclusterColdPostsMaxCandidates, + ContentExplorationSimclusterColdPostsPostsPerSimcluster, + EvergreenDRUserTweetMaxCandidates, + EvergreenDRCrossBorderUserTweetMaxCandidates, + DeepRetrievalTweetTweetANNMaxCandidates, + USSDeepRetrievalStratoTimeout, + OutlierDeepRetrievalStratoTimeout, + DeepRetrievalTweetTweetEmbeddingANNMaxCandidates, + DeepRetrievalTweetTweetEmbeddingRandomSize, + DeepRetrievalUserTweetANNMaxCandidates, + DeepRetrievalTweetTweetEmbeddingMinPriority, + MaxHaploliteTweetsParam, + MaxCandidateNumPerSourceKeyParam, + MediaDeepRetrievalUserTweetANNMaxCandidates, + MediaEvergreenDeepRetrievalUserTweetANNMaxCandidates, + MediaPromotedCreatorDeepRetrievalUserTweetANNMaxCandidates, + MediaRelatedCreatorMaxCandidates, + EvergreenVideosPaginationNumParam, + EvergreenVideosMaxFollowUsersParam, + TwhinRebuildTweetSimilarityMaxCandidates, + TwhinRebuildUserTweetSimilarityMaxCandidates, + ShortFormMinWidthParam, + ShortFormMinHeightParam, + ShortFormMaxResultParam, + LongFormMinWidthParam, + LongFormMinHeightParam, + LongFormMaxResultParam, + MemeMaxResultParam, + QigSearchHistoryTopKLimit, + MaxFollowersCountGateParam, + ) + + val enumFSOverrides = Seq(BlendingParam, TwhinRebuildTweetSimilarityDatasetParam) + + val booleanDeciderOverrides = Seq( + EnableHydraScoringSideEffect, + EnableEvergreenVideosSideEffect, + EnableDeepRetrievalAdhocDecider, + EnableTESMigrationDiffy + ) + + val boundedDoubleDeciderOverrides = Seq(DeepRetreivalAdhocMaxResultsDivByTenDecider) + + val booleanFSOverrides = Seq( + ControlAiEnabled, + ContentExplorationDRTweetTweetEnabled, + ContentExplorationDRTweetTweetEnableRandomEmbedding, + ContentExplorationEvergreenDRTweetTweetEnabled, + ContentExplorationDRTweetTweetTierTwoEnabled, + ContentExplorationEmbeddingSimilarityEnabled, + ContentExplorationMultimodalEnabled, + UserInterestSummarySimilarityEnabled, + ContentExplorationMediaEmbeddingSimilarityEnabled, + ContentExplorationNonVideoTweetFeaturesEnabled, + ContentExplorationEmbeddingSimilarityTierTwoEnabled, + ContentExplorationDRUserTweetEnabled, + ContentExplorationDRUserTweetTierTwoEnabled, + ContentExplorationSimclusterColdPostsEnabled, + LastNonPollingTimeFeatureHydratorEnabled, + DeepRetrievalTweetTweetANNEnabled, + DeepRetrievalTweetTweetEmbeddingANNEnabled, + DeepRetrievalUserTweetANNEnabled, + DeepRetrievalEnableGPU, + DeepRetrievalCategoricalUserTweetANNEnabled, + DeepRetrievalIsHighQualityEnabled, + DeepRetrievalIsLowNegEngRatioEnabled, + DeepRetrievalAddUserEmbeddingGaussianNoise, + EnableDebugMode, + EnableLowSignalUserCheck, + EnableMaxFollowersGate, + EnableNonEmptySearchHistoryUserCheck, + EnableTweetEntityServiceMigration, + EnableTweetEntityServiceMigrationDiffy, + EnableImpressionBloomFilterHydrator, + EnableGizmoduckQueryFeatureHydrator, + EnableVideoBloomFilterHydrator, + EnableShortVideoFilter, + EnableLongFormVideoFilter, + EnableShortFormVideoFilter, + EnableMediaIdDedupFilter, + EnableRelatedCreatorParam, + EnableMediaMetadataCandidateFeatureHydrator, + EnableMediaClusterIdDedupFilter, + EnableRequestCountryPlaceIdHydrator, + EnableSignalInfoHydrator, + EnableGrokFilter, + EnableUserTopicIdsHydrator, + EnableUSSFeatureHydrator, + EnableUSSGrokCategoryFeatureHydrator, + EnableUSSDeepRetrievalTweetEmbeddingFeatureHydrator, + EnableVideoTweetFilter, + EvergreenDRUserTweetEnabled, + EvergreenDRCrossBorderUserTweetEnabled, + HaploliteTweetsBasedEnabled, + HydraBasedSortingEnabled, + HydraScoringPipelineEnabled, + MediaDeepRetrievalUserTweetANNEnabled, + MediaDeepRetrievalIsHighQualityEnabled, + MediaDeepRetrievalIsLowNegEngRatioEnabled, + MediaEvergreenDeepRetrievalUserTweetANNEnabled, + MediaPromotedCreatorDeepRetrievalUserTweetANNEnabled, + DeepRetrievalNonVideoTweetFeaturesEnabled, + DeepRetrievalFilterOldSignalsEnabled, + SimclustersInterestedInEnabled, + SimclustersProducerBasedEnabled, + SimclustersTweetBasedEnabled, + SimclustersPromotedCreatorEnabled, + TwhinConsumerBasedEnabled, + TwhinRebuildTweetSimilarityEnabled, + TwhinRebuildUserTweetSimilarityEnabled, + TwhinTweetSimilarityEnabled, + UTEGEnabled, + UTGProducerBasedEnabled, + UTGTweetBasedEnabled, + UTGTweetBasedExpansionEnabled, + UVGTweetBasedEnabled, + UVGTweetBasedExpansionEnabled, + UVGHighQualityTweetBasedExpansionEnabled, + EnablePortraitVideoFilter, + EvergreenVideosEnabled, + GrokFilterFeatureHydratorEnabled, + ViewCountInfoOnTweetFeatureHydratorEnabled, + SemanticVideoCandidatePipelineEnabled, + TwitterClipV0LongVideoCandidatePipelineEnabled, + TwitterClipV0ShortVideoCandidatePipelineEnabled, + QigSearchHistoryTweetsEnabled, + QigSearchHistoryCandidateSourceEnabled, + QigSearchHistoryTweetsEnableLanguageFilter, + ScribeRetrievedCandidatesParam, + EnableContentEmbeddingAnnTweets, + ) + + val stringFSOverrides = Seq( + ExperimentBucketIdentifierParam, + DeepRetrievalModelName, + DeepRetrievalI2iEmbModelName, + USSDeepRetrievalI2iEmbModelName, + OutlierTweetEmbeddingModelNameParam, + DeepRetrievalVectorDBCollectionName, + DeepRetrievalI2iVectorDBCollectionName, + DeepRetrievalI2iEmbVectorDBCollectionName, + ContentExplorationDRTweetTweetVectorDBCollectionName, + ContentExplorationEvergreenDRTweetTweetVectorDBCollectionName, + ContentExplorationDRTweetTweetTierTwoVectorDBCollectionName, + ContentExplorationVectorDBCollectionName, + UserInterestSummarySimilarityVectorDBCollectionName, + ContentExplorationEmbeddingSimilarityTierTwoVectorDBCollectionName, + ContentExplorationDRUserTweetVectorDBCollectionName, + ContentExplorationDRUserEmbeddingModelName, + EvergreenDRUserEmbeddingModelName, + EvergreenDRUserTweetVectorDBCollectionName, + EvergreenDRCrossBorderUserEmbeddingModelName, + EvergreenDRCrossBorderUserTweetVectorDBCollectionName, + HydraModelName, + MediaDeepRetrievalModelName, + MediaEvergreenDeepRetrievalModelName, + MediaPromotedCreatorDeepRetrievalModelName, + MediaDeepRetrievalVectorDBCollectionName, + MediaEvergreenDeepRetrievalVectorDBCollectionName, + MediaPromotedCreatorDeepRetrievalVectorDBCollectionName, + TwhinRebuildUserTweetVectorDBName, + ) + + val boundedDoubleFSOverrides = Seq( + ControlAiShowMoreWeightParam, + ControlAiShowLessWeightParam, + ContentExplorationDRTweetTweetScoreThreshold, + ContentExplorationEvergreenDRTweetTweetScoreThreshold, + ContentExplorationDRTweetTweetTierTwoScoreThreshold, + ContentExplorationEmbeddingANNScoreThreshold, + UserInterestSummarySimilarityScoreThreshold, + ContentExplorationEmbeddingSimilarityTierTwoScoreThreshold, + DeepRetrievalUserEmbeddingGaussianNoiseParam, + DeepRetrievalTweetTweetANNScoreThreshold, + USSDeepRetrievalSimilarityTweetTweetANNScoreThreshold, + DeepRetrievalUserTweetANNScoreThreshold, + DeepRetrievalTweetTweetEmbeddingANNScoreThreshold, + DeepRetrievalTweetTweetANNScoreMaxThreshold, + DeepRetrievalTweetTweetEmbeddingTimeDecay, + DeepRetrievalTweetTweetEmbeddingTimeOfDayBoostWeight, + DeepRetrievalTweetTweetEmbeddingDayOfWeekBoostWeight, + DeepRetrievalTweetTweetEmbeddingEngagementBoostWeight, + ReplyScoreWeightParam, + VideoScoreWeightParam, + ShortFormMinAspectRatioParam, + ShortFormMaxAspectRatioParam, + LongFormMinAspectRatioParam, + LongFormMaxAspectRatioParam, + LowSignalUserGrokTopicRatioParam, + LowSignalUserPopGeoRatioParam, + LowSignalUserSimclustersRatioParam + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/USSParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/USSParams.scala new file mode 100644 index 000000000..1d78e904f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/USSParams.scala @@ -0,0 +1,575 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.timelines.configapi.DurationConversion +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSEnumParam +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.HasDurationConversion +import com.twitter.usersignalservice.thriftscala.SignalType +import com.twitter.util.Duration + +/** + * USS Store Related Params + */ +object USSParams { + + /** + * Enabling Signals Params + */ + object EnableRecentTweetFavorites + extends FSParam[Boolean]( + name = "uss_enable_recent_tweet_favorites", + default = false + ) + + object EnableRecentRetweets + extends FSParam[Boolean]( + name = "uss_enable_recent_retweets", + default = false + ) + + object EnableRecentReplies + extends FSParam[Boolean]( + name = "uss_enable_recent_replies", + default = false + ) + + object EnableRecentTweetBookmarks + extends FSParam[Boolean]( + name = "uss_enable_recent_tweet_bookmarks", + default = false + ) + + object EnableRecentTweetFeedbackRelevant + extends FSParam[Boolean]( + name = "uss_enable_recent_tweet_feedback_relevant", + default = false + ) + + object EnableRecentTweetFeedbackNotrelevant + extends FSParam[Boolean]( + name = "uss_enable_recent_tweet_feedback_notrelevant", + default = false + ) + + object EnableRecentOriginalTweets + extends FSParam[Boolean]( + name = "uss_enable_recent_original_tweets", + default = false + ) + + object EnableRecentFollows + extends FSParam[Boolean]( + name = "uss_enable_recent_follows", + default = false + ) + + object EnableRepeatedProfileVisits + extends FSParam[Boolean]( + name = "uss_enable_repeated_profile_visits", + default = false + ) + + object EnableRecentNotifications + extends FSParam[Boolean]( + name = "uss_enable_recent_notifications", + default = false + ) + + object EnableTweetShares + extends FSParam[Boolean]( + name = "uss_enable_tweet_shares", + default = false + ) + + object EnableTweetDetailGoodClick1Min + extends FSParam[Boolean]( + name = "uss_enable_tweet_detail_good_click_1_min", + default = false + ) + + object EnableVideoViewTweets // This is the cumulative of the bottom 3 signals + home + extends FSParam[Boolean]( + name = "uss_enable_video_view_tweets", + default = false + ) + + object EnableVideoViewVisibilityFilteredTweets + extends FSParam[Boolean]( + name = "uss_enable_video_view_visibility_filtered_tweets", + default = false + ) + + object EnableVideoViewVisibility75FilteredTweets + extends FSParam[Boolean]( + name = "uss_enable_video_view_visibility_75_filtered_tweets", + default = false + ) + + object EnableVideoViewVisibility100FilteredTweets + extends FSParam[Boolean]( + name = "uss_enable_video_view_visibility_100_filtered_tweets", + default = false + ) + + object EnableVideoViewHighResolutionFilteredTweets + extends FSParam[Boolean]( + name = "uss_enable_video_view_high_resolution_filtered_tweets", + default = false + ) + + object EnableImmersiveVideoViewTweets + extends FSParam[Boolean]( + name = "uss_enable_immersive_video_view_tweets", + default = false + ) + + object EnableMediaImmersiveVideoViewTweets + extends FSParam[Boolean]( + name = "uss_enable_media_immersive_video_view_tweets", + default = false + ) + + object EnableTvVideoViewTweets + extends FSParam[Boolean]( + name = "uss_enable_tv_video_view_tweets", + default = false + ) + + object EnableWatchTimeTweets // This is cumulative of the below 3 signals + extends FSParam[Boolean]( + name = "uss_enable_watch_time_tweets", + default = false + ) + + object EnableImmersiveWatchTimeTweets + extends FSParam[Boolean]( + name = "uss_enable_immersive_watch_time_tweets", + default = false + ) + + object EnableMediaImmersiveWatchTimeTweets + extends FSParam[Boolean]( + name = "uss_enable_media_immersive_watch_time_tweets", + default = false + ) + + object EnableTvWatchTimeTweets + extends FSParam[Boolean]( + name = "uss_enable_tv_watch_time_tweets", + default = false + ) + + object EnableSearcherRealtimeHistory + extends FSParam[Boolean]( + name = "uss_enable_searcher_realtime_history", + default = false + ) + + object EnableNegativeSignals + extends FSParam[Boolean]( + name = "uss_enable_negative_signals", + default = false + ) + + object EnableHighQualitySourceTweet + extends FSParam[Boolean]( + name = "uss_enable_high_quality_source_tweet", + default = false + ) + + object EnableHighQualitySourceUser + extends FSParam[Boolean]( + name = "uss_enable_high_quality_source_user", + default = false + ) + + object EnableTweetPhotoExpand + extends FSParam[Boolean]( + name = "uss_enable_tweet_photo_expand", + default = false + ) + + object EnableSearchTweetClick + extends FSParam[Boolean]( + name = "uss_enable_search_tweet_click", + default = false + ) + + object EnableProfileTweetClick + extends FSParam[Boolean]( + name = "uss_enable_profile_tweet_click", + default = false + ) + + object EnableTweetVideoOpen + extends FSParam[Boolean]( + name = "uss_enable_tweet_video_open", + default = false + ) + + /** + * Unified params for all types of signals to fetch + */ + object UnifiedMaxSourceKeyNum + extends FSBoundedParam[Int]( + name = "uss_unified_max_source_key_num", + default = 15, + min = 0, + max = 100 + ) + + object UnifiedMinSignalFavCount + extends FSBoundedParam[Int]( + name = "uss_unified_min_signal_fav_count", + default = 10, + min = 0, + max = Int.MaxValue + ) + + object UnifiedMaxSignalFavCount + extends FSBoundedParam[Int]( + name = "uss_unified_max_signal_fav_count", + default = Int.MaxValue, + min = 0, + max = Int.MaxValue + ) + + object UnifiedMaxSignalAgeInHours + extends FSBoundedParam[Int]( + name = "uss_unified_max_signal_age_in_hours", + default = Int.MaxValue, + min = 0, + max = Int.MaxValue + ) + + object VQVMaxSignalAgeInDays + extends FSBoundedParam[Int]( + name = "uss_vqv_max_signal_age_in_days", + default = Int.MaxValue, + min = 0, + max = Int.MaxValue + ) + + object LowSignalUserMaxSignalCount + extends FSBoundedParam[Int]( + name = "uss_low_signal_user_max_signal_count", + default = 15, + min = 0, + max = Int.MaxValue + ) + + object LowSignalUserMaxSignalAge + extends FSBoundedParam[Duration]( + "uss_low_signal_user_max_signal_age_in_days", + default = 90.days, + min = 1.days, + max = 1000.days) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromDays + } + + object EnableNegativeSentimentSignalFilter + extends FSParam[Boolean]( + "uss_negative_sentiment_signal_filter", + default = false + ) + + object EnableNegativeSourceSignal + extends FSParam[Boolean]( + "uss_enable_negative_source_signal", + default = false + ) + + object MaxNegativeSourceSignals + extends FSBoundedParam[Int]( + name = "uss_max_negative_source_signals", + default = 15, + min = 0, + max = 200 + ) + + object EnableNegativeSourceSignalBucketFilter + extends FSParam[Boolean]( + name = "uss_enable_negative_source_signal_bucket_filter", + default = false + ) + + object NegativeSourceSignalEligible + extends FSBoundedParam[Int]( + name = "uss_negative_source_signal_eligible", + default = 2, + min = 0, + max = 100 + ) + + object MaxHighQualitySourceSignals + extends FSBoundedParam[Int]( + name = "uss_max_high_quality_source_signals", + default = 15, + min = 0, + max = 50 + ) + + object EnableHighQualitySourceSignalBucketFilter + extends FSParam[Boolean]( + name = "uss_enable_high_quality_source_signal_bucket_filter", + default = false + ) + + object HighQualitySourceTweetEligible + extends FSBoundedParam[Int]( + name = "uss_high_quality_source_tweet_eligible", + default = 0, + min = 0, + max = 100 + ) + + object HighQualitySourceUserEligible + extends FSBoundedParam[Int]( + name = "uss_high_quality_source_user_eligible", + default = 0, + min = 0, + max = 100 + ) + + object MaxVideoViewSourceSignals + extends FSBoundedParam[Int]( + name = "uss_max_video_view_source_signals", + default = 30, + min = 0, + max = 100 + ) + + object MaxFavSignals + extends FSBoundedParam[Int]( + name = "uss_max_fav_signals", + default = 15, + min = 0, + max = 50 + ) + + object MaxBookmarkSignals + extends FSBoundedParam[Int]( + name = "uss_max_bookmark_signals", + default = 15, + min = 0, + max = 50 + ) + + object MaxVqvSignals + extends FSBoundedParam[Int]( + name = "uss_max_vqv_signals", + default = 15, + min = 0, + max = 50 + ) + + object MaxPhotoExpandSignals + extends FSBoundedParam[Int]( + name = "uss_max_photo_expand_signals", + default = 15, + min = 0, + max = 50 + ) + + object MaxSearchTweetClick + extends FSBoundedParam[Int]( + name = "uss_max_search_tweet_click_signals", + default = 0, + min = 0, + max = 50 + ) + + object MaxProfileTweetClick + extends FSBoundedParam[Int]( + name = "uss_max_profile_tweet_click_signals", + default = 0, + min = 0, + max = 50 + ) + + object MaxTweetVideoOpen + extends FSBoundedParam[Int]( + name = "uss_max_tweet_video_open_signals", + default = 0, + min = 0, + max = 50 + ) + + /** + * Type of signals to fetch related params + */ + object ProfileMinVisitEnum extends Enumeration { + + val TotalVisitsInPast180Days = Value + val TotalVisitsInPast90Days = Value + val TotalVisitsInPast14Days = Value + val TotalVisitsInPast180DaysNoNegative = Value + val TotalVisitsInPast90DaysNoNegative = Value + val TotalVisitsInPast14DaysNoNegative = Value + } + + object VideoViewTweetTypeEnum extends Enumeration { + val VideoTweetQualityView = Value + val VideoTweetPlayback50 = Value + val VideoTweetQualityViewAllSurfaces = Value + val VideoTweetQualityViewV2 = Value + val VideoTweetQualityViewV2Visibility75 = Value + val VideoTweetQualityViewV2Visibility100 = Value + val VideoTweetQualityViewV3 = Value + } + + object ProfileMinVisitType + extends FSEnumParam[ProfileMinVisitEnum.type]( + name = "uss_profile_min_visit_type_id", + default = ProfileMinVisitEnum.TotalVisitsInPast180Days, + enum = ProfileMinVisitEnum + ) + + object VideoViewTweetTypeParam + extends FSEnumParam[VideoViewTweetTypeEnum.type]( + name = "uss_video_view_tweet_type_id", + default = VideoViewTweetTypeEnum.VideoTweetQualityView, + enum = VideoViewTweetTypeEnum + ) + + object VideoViewVisibilityFilteredTweetTypeParam + extends FSEnumParam[VideoViewTweetTypeEnum.type]( + name = "uss_video_view_visibility_filtered_tweet_type_id", + default = VideoViewTweetTypeEnum.VideoTweetQualityViewV2, + enum = VideoViewTweetTypeEnum + ) + + object VideoViewVisibility75FilteredTweetTypeParam + extends FSEnumParam[VideoViewTweetTypeEnum.type]( + name = "uss_video_view_visibility_75_filtered_tweet_type_id", + default = VideoViewTweetTypeEnum.VideoTweetQualityViewV2Visibility75, + enum = VideoViewTweetTypeEnum + ) + + object VideoViewVisibility100FilteredTweetTypeParam + extends FSEnumParam[VideoViewTweetTypeEnum.type]( + name = "uss_video_view_visibility_100_filtered_tweet_type_id", + default = VideoViewTweetTypeEnum.VideoTweetQualityViewV2Visibility100, + enum = VideoViewTweetTypeEnum + ) + + object VideoViewHighResolutionFilteredTweetTypeParam + extends FSEnumParam[VideoViewTweetTypeEnum.type]( + name = "uss_video_view_high_resolution_filtered_tweet_type_id", + default = VideoViewTweetTypeEnum.VideoTweetQualityViewV3, + enum = VideoViewTweetTypeEnum + ) + + def profileMinVisitParam(profileMinVisitEnum: ProfileMinVisitEnum.Value): SignalType = { + profileMinVisitEnum match { + case ProfileMinVisitEnum.TotalVisitsInPast180Days => + SignalType.RepeatedProfileVisit180dMinVisit6V1 + case ProfileMinVisitEnum.TotalVisitsInPast90Days => + SignalType.RepeatedProfileVisit90dMinVisit6V1 + case ProfileMinVisitEnum.TotalVisitsInPast14Days => + SignalType.RepeatedProfileVisit14dMinVisit2V1 + case ProfileMinVisitEnum.TotalVisitsInPast180DaysNoNegative => + SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative + case ProfileMinVisitEnum.TotalVisitsInPast90DaysNoNegative => + SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative + case ProfileMinVisitEnum.TotalVisitsInPast14DaysNoNegative => + SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative + } + } + + def videoViewTweetTypeParam(videoViewTweetTypeEnum: VideoViewTweetTypeEnum.Value): SignalType = { + videoViewTweetTypeEnum match { + case VideoViewTweetTypeEnum.VideoTweetQualityView => SignalType.VideoView90dQualityV1 + case VideoViewTweetTypeEnum.VideoTweetPlayback50 => SignalType.VideoView90dPlayback50V1 + case VideoViewTweetTypeEnum.VideoTweetQualityViewAllSurfaces => + SignalType.VideoView90dQualityV1AllSurfaces + case VideoViewTweetTypeEnum.VideoTweetQualityViewV2 => SignalType.VideoView90dQualityV2 + case VideoViewTweetTypeEnum.VideoTweetQualityViewV2Visibility75 => + SignalType.VideoView90dQualityV2Visibility75 + case VideoViewTweetTypeEnum.VideoTweetQualityViewV2Visibility100 => + SignalType.VideoView90dQualityV2Visibility100 + case VideoViewTweetTypeEnum.VideoTweetQualityViewV3 => SignalType.VideoView90dQualityV3 + } + } + + val booleanFSOverrides = + Seq( + EnableRecentTweetFavorites, + EnableRecentRetweets, + EnableRecentReplies, + EnableRecentTweetBookmarks, + EnableRecentTweetFeedbackRelevant, + EnableRecentTweetFeedbackNotrelevant, + EnableRecentOriginalTweets, + EnableRecentFollows, + EnableRepeatedProfileVisits, + EnableTweetShares, + EnableTweetPhotoExpand, + EnableSearchTweetClick, + EnableProfileTweetClick, + EnableTweetVideoOpen, + EnableTweetDetailGoodClick1Min, + EnableVideoViewTweets, + EnableVideoViewVisibilityFilteredTweets, + EnableVideoViewVisibility75FilteredTweets, + EnableVideoViewVisibility100FilteredTweets, + EnableVideoViewHighResolutionFilteredTweets, + EnableImmersiveVideoViewTweets, + EnableMediaImmersiveVideoViewTweets, + EnableTvVideoViewTweets, + EnableWatchTimeTweets, + EnableImmersiveWatchTimeTweets, + EnableMediaImmersiveWatchTimeTweets, + EnableTvWatchTimeTweets, + EnableNegativeSignals, + EnableRecentNotifications, + EnableSearcherRealtimeHistory, + EnableNegativeSentimentSignalFilter, + EnableNegativeSourceSignal, + EnableNegativeSourceSignalBucketFilter, + EnableHighQualitySourceTweet, + EnableHighQualitySourceUser, + EnableHighQualitySourceSignalBucketFilter, + ) + + val boundedIntFSOverrides = + Seq( + UnifiedMaxSourceKeyNum, + UnifiedMinSignalFavCount, + UnifiedMaxSignalFavCount, + UnifiedMaxSignalAgeInHours, + LowSignalUserMaxSignalCount, + VQVMaxSignalAgeInDays, + MaxNegativeSourceSignals, + NegativeSourceSignalEligible, + MaxHighQualitySourceSignals, + HighQualitySourceTweetEligible, + HighQualitySourceUserEligible, + MaxVideoViewSourceSignals, + MaxFavSignals, + MaxBookmarkSignals, + MaxVqvSignals, + MaxPhotoExpandSignals, + MaxSearchTweetClick, + MaxProfileTweetClick, + MaxTweetVideoOpen, + ) + + val enumFSOverrides = + Seq( + ProfileMinVisitType, + VideoViewTweetTypeParam, + VideoViewVisibilityFilteredTweetTypeParam, + VideoViewVisibility75FilteredTweetTypeParam, + VideoViewVisibility100FilteredTweetTypeParam, + VideoViewHighResolutionFilteredTweetTypeParam + ) + + val boundedDurationFSOverrides = + Seq( + LowSignalUserMaxSignalAge + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UTGParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UTGParams.scala new file mode 100644 index 000000000..76a7880ce --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UTGParams.scala @@ -0,0 +1,142 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.recos.user_tweet_graph.thriftscala.RelatedTweetSimilarityAlgorithm +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSEnumParam +import com.twitter.timelines.configapi.FSParam + +//UTG Related Params +object UTGParams { + + object SimilarityAlgorithmEnum extends Enumeration { + val Cosine, LogCosine, PowerDegree, PowerDegreeWithModifiedScorePreFactor, + CoocurrenceNormLogDegree, CoocurrenceNormLinearDegree, CoocurrenceNormPowerDegree, + LogBiasCoocurrenceNormLogDegree, LogBiasCoocurrenceNormPowerDegree: Value = Value + val enumToSimilarityAlgorithmMap: Map[ + SimilarityAlgorithmEnum.Value, + RelatedTweetSimilarityAlgorithm + ] = Map( + Cosine -> RelatedTweetSimilarityAlgorithm.Cosine, + LogCosine -> RelatedTweetSimilarityAlgorithm.LogCosine, + PowerDegree -> RelatedTweetSimilarityAlgorithm.PowerDegree, + PowerDegreeWithModifiedScorePreFactor -> RelatedTweetSimilarityAlgorithm.PowerDegreeWithModifiedScorePreFactor, + CoocurrenceNormLogDegree -> RelatedTweetSimilarityAlgorithm.CoocurrenceNormLogDegree, + CoocurrenceNormLinearDegree -> RelatedTweetSimilarityAlgorithm.CoocurrenceNormLinearDegree, + CoocurrenceNormPowerDegree -> RelatedTweetSimilarityAlgorithm.CoocurrenceNormPowerDegree, + LogBiasCoocurrenceNormLogDegree -> RelatedTweetSimilarityAlgorithm.LogBiasCoocurrenceNormLogDegree, + LogBiasCoocurrenceNormPowerDegree -> RelatedTweetSimilarityAlgorithm.LogBiasCoocurrenceNormPowerDegree + ) + } + object MinCoOccurrenceParam + extends FSBoundedParam[Int]( + name = "user_tweet_graph_min_co_occurrence", + default = 3, + min = 0, + max = 500 + ) + + object MaxNumFollowersParam + extends FSBoundedParam[Int]( + name = "user_tweet_graph_max_num_followers", + default = 500, + min = 100, + max = 1000 + ) + + object TweetBasedMinScoreParam + extends FSBoundedParam[Double]( + name = "user_tweet_graph_tweet_based_min_score", + default = 0.5, + min = 0.0, + max = 10.0 + ) + + object TweetBasedDegreeExponentParam + extends FSBoundedParam[Double]( + name = "user_tweet_graph_tweet_based_degree_exponent", + default = 0.5, + min = 0.1, + max = 1.0 + ) + + object ConsumersBasedMinScoreParam + extends FSBoundedParam[Double]( + name = "user_tweet_graph_consumers_based_min_score", + default = 4.0, + min = 0.0, + max = 10.0 + ) + + object MaxConsumerSeedsNumParam + extends FSBoundedParam[Int]( + name = "user_tweet_graph_max_user_seeds_num", + default = 100, + min = 0, + max = 300 + ) + + object CoverageExpansionOldTweetEnabledParam + extends FSParam[Boolean]( + name = "user_tweet_graph_coverage_expansion_old_tweet_enabled", + default = false + ) + + object EnableUTGCacheParam + extends FSParam[Boolean]( + name = "user_tweet_graph_enable_cache", + default = true + ) + + object SimilarityAlgorithm + extends FSEnumParam[SimilarityAlgorithmEnum.type]( + name = "user_tweet_graph_similarity_algorithm_id", + default = SimilarityAlgorithmEnum.LogCosine, + enum = SimilarityAlgorithmEnum + ) + + object EnableTweetEmbeddingBasedFilteringParam + extends FSParam[Boolean]( + name = "user_tweet_graph_enable_tweet_embedding_based_filtering", + default = false + ) + + object OutlierFilterPercentileThresholdParam + extends FSBoundedParam[Int]( + name = "user_tweet_graph_outlier_filter_percentile_threshold", + default = 80, + min = 0, + max = 100 + ) + + object OutlierMinRequiredSignalsParam + extends FSBoundedParam[Int]( + name = "user_tweet_graph_outlier_min_required_signals", + default = 10, + min = 0, + max = 100 + ) + + val booleanFSOverrides = + Seq( + CoverageExpansionOldTweetEnabledParam, + EnableUTGCacheParam, + EnableTweetEmbeddingBasedFilteringParam, + ) + + val boundedIntFSOverrides = + Seq( + MinCoOccurrenceParam, + MaxConsumerSeedsNumParam, + OutlierFilterPercentileThresholdParam, + OutlierMinRequiredSignalsParam + ) + + val boundedDoubleFSOverrides = + Seq( + TweetBasedMinScoreParam, + ConsumersBasedMinScoreParam, + TweetBasedDegreeExponentParam + ) + + val enumFSOverrides = Seq(SimilarityAlgorithm) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UVGParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UVGParams.scala new file mode 100644 index 000000000..bd67d8627 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UVGParams.scala @@ -0,0 +1,176 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.recos.user_video_graph.thriftscala.RelatedTweetSimilarityAlgorithm +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSEnumParam +import com.twitter.timelines.configapi.FSParam + +//UVG Related Params +object UVGParams { + + object SimilarityAlgorithmEnum extends Enumeration { + val Cosine, LogCosine, PowerDegree, PowerDegreeWithModifiedScorePreFactor, + CoocurrenceNormLogDegree, CoocurrenceNormLinearDegree, CoocurrenceNormPowerDegree, + LogBiasCoocurrenceNormLogDegree, LogBiasCoocurrenceNormPowerDegree: Value = Value + val enumToSimilarityAlgorithmMap: Map[ + SimilarityAlgorithmEnum.Value, + RelatedTweetSimilarityAlgorithm + ] = Map( + Cosine -> RelatedTweetSimilarityAlgorithm.Cosine, + LogCosine -> RelatedTweetSimilarityAlgorithm.LogCosine, + PowerDegree -> RelatedTweetSimilarityAlgorithm.PowerDegree, + PowerDegreeWithModifiedScorePreFactor -> RelatedTweetSimilarityAlgorithm.PowerDegreeWithModifiedScorePreFactor, + CoocurrenceNormLogDegree -> RelatedTweetSimilarityAlgorithm.CoocurrenceNormLogDegree, + CoocurrenceNormLinearDegree -> RelatedTweetSimilarityAlgorithm.CoocurrenceNormLinearDegree, + CoocurrenceNormPowerDegree -> RelatedTweetSimilarityAlgorithm.CoocurrenceNormPowerDegree, + LogBiasCoocurrenceNormLogDegree -> RelatedTweetSimilarityAlgorithm.LogBiasCoocurrenceNormLogDegree, + LogBiasCoocurrenceNormPowerDegree -> RelatedTweetSimilarityAlgorithm.LogBiasCoocurrenceNormPowerDegree + ) + } + + object MinCoOccurrenceParam + extends FSBoundedParam[Int]( + name = "user_video_graph_min_co_occurrence", + default = 3, + min = 0, + max = 500 + ) + + object TweetBasedMinScoreParam + extends FSBoundedParam[Double]( + name = "user_video_graph_tweet_based_min_score", + default = 0.5, + min = 0.0, + max = 10.0 + ) + object TweetBasedDegreeExponentParam + extends FSBoundedParam[Double]( + name = "user_video_graph_tweet_based_degree_exponent", + default = 0.5, + min = 0.1, + max = 1.0 + ) + object ConsumersBasedMinScoreParam + extends FSBoundedParam[Double]( + name = "user_video_graph_consumers_based_min_score", + default = 4.0, + min = 0.0, + max = 10.0 + ) + + object MaxConsumerSeedsNumParam + extends FSBoundedParam[Int]( + name = "user_video_graph_max_user_seeds_num", + default = 100, + min = 0, + max = 300 + ) + + object CoverageExpansionOldTweetEnabledParam + extends FSParam[Boolean]( + name = "user_video_graph_coverage_expansion_old_tweet_enabled", + default = false + ) + + object SimilarityAlgorithm + extends FSEnumParam[SimilarityAlgorithmEnum.type]( + name = "user_video_graph_similarity_algorithm_id", + default = SimilarityAlgorithmEnum.LogCosine, + enum = SimilarityAlgorithmEnum + ) + + object EnableUVGCacheParam + extends FSParam[Boolean]( + name = "user_video_graph_enable_cache", + default = true + ) + + /** + * Sets the number of users to sample from the input video, this increases fanout significantly + */ + object MaxNumSamplesPerNeighborParam + extends FSBoundedParam[Int]( + name = "user_video_graph_max_num_samples_per_neighbor", + default = 100, + min = 100, + max = 100000 + ) + + /** + * Filters out spammy users we sample for related queries, i.e. don't sample users that watch over K videos + */ + object MaxLeftNodeDegreeParam + extends FSBoundedParam[Int]( + name = "user_video_graph_max_left_node_degree", + default = 100, + min = 0, + max = 100000 + ) + + /** + * Filters out videos with too many views, may pollute related results + */ + object MaxRightNodeDegreeParam + extends FSBoundedParam[Int]( + name = "user_video_graph_max_right_node_degree", + default = 0, // 0 maps to no limit in UVG + min = 0, + max = Int.MaxValue + ) + + object SampleRHSTweetsParam + extends FSParam[Boolean]( + name = "user_video_graph_sample_rhs_tweets", + default = false + ) + + object EnableTweetEmbeddingBasedFilteringParam + extends FSParam[Boolean]( + name = "user_video_graph_enable_tweet_embedding_based_filtering", + default = false + ) + + object OutlierFilterPercentileThresholdParam + extends FSBoundedParam[Int]( + name = "user_video_graph_outlier_filter_percentile_threshold", + default = 80, + min = 0, + max = 100 + ) + + object OutlierMinRequiredSignalsParam + extends FSBoundedParam[Int]( + name = "user_video_graph_outlier_min_required_signals", + default = 10, + min = 0, + max = 100 + ) + + val booleanFSOverrides = + Seq( + CoverageExpansionOldTweetEnabledParam, + EnableUVGCacheParam, + SampleRHSTweetsParam, + EnableTweetEmbeddingBasedFilteringParam + ) + + val boundedIntFSOverrides = + Seq( + MinCoOccurrenceParam, + MaxConsumerSeedsNumParam, + MaxNumSamplesPerNeighborParam, + MaxLeftNodeDegreeParam, + MaxRightNodeDegreeParam, + OutlierFilterPercentileThresholdParam, + OutlierMinRequiredSignalsParam + ) + + val boundedDoubleFSOverrides = + Seq( + TweetBasedMinScoreParam, + ConsumersBasedMinScoreParam, + TweetBasedDegreeExponentParam + ) + + val enumFSOverrides = Seq(SimilarityAlgorithm) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UserLocationParams.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UserLocationParams.scala new file mode 100644 index 000000000..d908e663a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/UserLocationParams.scala @@ -0,0 +1,13 @@ +package com.twitter.tweet_mixer.param + +import com.twitter.timelines.configapi.FSParam + +object UserLocationParams { + val booleanFSOverrides = Seq(UserLocationEnabled) + + object UserLocationEnabled + extends FSParam[Boolean]( + name = "tweet_mixer_user_location_enabled", + default = false + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/decider/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/decider/BUILD.bazel new file mode 100644 index 000000000..24c5a2b44 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/decider/BUILD.bazel @@ -0,0 +1,9 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "servo/decider", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/decider/DeciderKey.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/decider/DeciderKey.scala new file mode 100644 index 000000000..15cc0daed --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/decider/DeciderKey.scala @@ -0,0 +1,23 @@ +package com.twitter.tweet_mixer.param.decider + +import com.twitter.servo.decider.DeciderKeyEnum + +object DeciderKey extends DeciderKeyEnum { + val EnableHomeRecommendedTweetsProduct = Value("enable_home_recommended_tweets_product") + val EnableNotificationsRecommendedTweetsProduct = Value( + "enable_notifications_recommended_tweets_product") + val EnableIMVRecommendedTweetsProduct = Value("enable_imv_recommended_tweets_product") + val EnableIMVRelatedTweetsProduct = Value("enable_imv_related_tweets_product") + val EnableRUXRelatedTweetsProduct = Value("enable_rux_related_tweets_product") + val EnableDebuggerTweetsProduct = Value("enable_debugger_tweets_product") + val EnableVideoRecommendedTweetsProduct = Value("enable_video_recommended_tweets_product") + val EnableHydraScoringSideEffect = Value("enable_hydra_scoring_side_effect") + val EnableEvergreenVideosSideEffect = Value("enable_evergreen_videos_side_effect") + val EnableLoggedOutVideoRecommendedTweetsProduct = Value( + "enable_logged_out_video_recommended_tweets_product") + val EnableTopicTweetsProduct = Value("enable_topic_tweets_product") + val EnableDeepRetrievalAdhocSideEffect = Value("enable_deep_retrieval_adhoc_side_effect") + val DeepRetrievalSideEffectNumCandidatesDivByTen = Value( + "deep_retrieval_side_effect_num_candidates_div_by_10") + val EnableTESMigrationDiffy = Value("enable_tes_migration_diffy") +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/BUILD.bazel new file mode 100644 index 000000000..cf14e9fb6 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/BUILD.bazel @@ -0,0 +1,19 @@ +scala_library( + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/debugger_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/imv_recommended_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/imv_related_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/logged_out_video_recommended_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/notifications_recommended_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/rux_related_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/topic_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/video_recommended_tweets", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/TweetMixerProductModule.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/TweetMixerProductModule.scala new file mode 100644 index 000000000..c393f5bb2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/TweetMixerProductModule.scala @@ -0,0 +1,11 @@ +package com.twitter.tweet_mixer.product + +import com.twitter.inject.TwitterModule +import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistryConfig + +object TweetMixerProductModule extends TwitterModule { + + override def configure(): Unit = { + bind[ProductPipelineRegistryConfig].to[TweetMixerProductPipelineRegistryConfig] + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/TweetMixerProductPipelineRegistryConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/TweetMixerProductPipelineRegistryConfig.scala new file mode 100644 index 000000000..6bd5e72a9 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/TweetMixerProductPipelineRegistryConfig.scala @@ -0,0 +1,90 @@ +package com.twitter.tweet_mixer.product + +import com.twitter.inject.Injector +import com.twitter.product_mixer.core.product.guice.ProductScope +import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistryConfig +import com.twitter.tweet_mixer.model.request.DebuggerTweetsProduct +import com.twitter.tweet_mixer.model.request.HomeRecommendedTweetsProduct +import com.twitter.tweet_mixer.model.request.IMVRecommendedTweetsProduct +import com.twitter.tweet_mixer.model.request.IMVRelatedTweetsProduct +import com.twitter.tweet_mixer.model.request.LoggedOutVideoRecommendedTweetsProduct +import com.twitter.tweet_mixer.model.request.NotificationsRecommendedTweetsProduct +import com.twitter.tweet_mixer.model.request.RUXRelatedTweetsProduct +import com.twitter.tweet_mixer.model.request.TopicTweetsProduct +import com.twitter.tweet_mixer.model.request.VideoRecommendedTweetsProduct +import com.twitter.tweet_mixer.product.home_recommended_tweets.HomeRecommendedTweetsProductPipelineConfig +import com.twitter.tweet_mixer.product.imv_recommended_tweets.IMVRecommendedTweetsProductPipelineConfig +import com.twitter.tweet_mixer.product.topic_tweets.TopicTweetsProductPipelineConfig +import com.twitter.tweet_mixer.product.imv_related_tweets.IMVRelatedTweetsProductPipelineConfig +import com.twitter.tweet_mixer.product.logged_out_video_recommended_tweets.LoggedOutVideoRecommendedTweetsProductPipelineConfig +import com.twitter.tweet_mixer.product.notifications_recommended_tweets.NotificationsRecommendedTweetsProductPipelineConfig +import com.twitter.tweet_mixer.product.rux_related_tweets.RUXRelatedTweetsProductPipelineConfig +import com.twitter.tweet_mixer.product.debugger_tweets.DebuggerTweetsProductPipelineConfig +import com.twitter.tweet_mixer.product.video_recommended_tweets.VideoRecommendedTweetsProductPipelineConfig + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TweetMixerProductPipelineRegistryConfig @Inject() ( + injector: Injector, + productScope: ProductScope) + extends ProductPipelineRegistryConfig { + + private val homeRecommendedTweetsProductPipelineConfig = + productScope.let(HomeRecommendedTweetsProduct) { + injector.instance[HomeRecommendedTweetsProductPipelineConfig] + } + + private val notificationsRecommendedTweetsProductPipelineConfig = + productScope.let(NotificationsRecommendedTweetsProduct) { + injector.instance[NotificationsRecommendedTweetsProductPipelineConfig] + } + + private val imvRecommendedTweetsProductPipelineConfig = + productScope.let(IMVRecommendedTweetsProduct) { + injector.instance[IMVRecommendedTweetsProductPipelineConfig] + } + + private val imvRelatedTweetsProductPipelineConfig = + productScope.let(IMVRelatedTweetsProduct) { + injector.instance[IMVRelatedTweetsProductPipelineConfig] + } + + private val ruxRelatedTweetsProductPipelineConfig = + productScope.let(RUXRelatedTweetsProduct) { + injector.instance[RUXRelatedTweetsProductPipelineConfig] + } + + private val debuggerTweetsProductPipelineConfig = + productScope.let(DebuggerTweetsProduct) { + injector.instance[DebuggerTweetsProductPipelineConfig] + } + + private val videoRecommendedTweetsProductPipelineConfig = + productScope.let(VideoRecommendedTweetsProduct) { + injector.instance[VideoRecommendedTweetsProductPipelineConfig] + } + + private val loggedOutVideoRecommendedTweetsProductPipelineConfig = + productScope.let(LoggedOutVideoRecommendedTweetsProduct) { + injector.instance[LoggedOutVideoRecommendedTweetsProductPipelineConfig] + } + + private val topicTweetsProductPipelineConfig = + productScope.let(TopicTweetsProduct) { + injector.instance[TopicTweetsProductPipelineConfig] + } + + override val productPipelineConfigs = Seq( + homeRecommendedTweetsProductPipelineConfig, + notificationsRecommendedTweetsProductPipelineConfig, + imvRecommendedTweetsProductPipelineConfig, + imvRelatedTweetsProductPipelineConfig, + ruxRelatedTweetsProductPipelineConfig, + videoRecommendedTweetsProductPipelineConfig, + loggedOutVideoRecommendedTweetsProductPipelineConfig, + topicTweetsProductPipelineConfig, + debuggerTweetsProductPipelineConfig + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/BUILD.bazel new file mode 100644 index 000000000..945117f7a --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/BUILD.bazel @@ -0,0 +1,62 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/location", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", + "servo/repo", + "src/scala/com/twitter/simclusters_v2/common", + "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", + "src/thrift/com/twitter/search:earlybird-scala", + "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", + "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", + "timelineservice/common:model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UTG", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/UVG", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/content_embedding_ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/earlybird_realtime_cg", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/evergreen_videos", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/ndr_ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_geo_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_grok_topic_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/popular_topic_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/qig_service", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/simclusters_ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/text_embedding_ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/topic_tweets", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_source/twhin_ann", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/config", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/filter", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/gate", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/selector", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/side_effect", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/transformer", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scorer", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scoring_pipeline", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], + exports = [ + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/HomeRecommendedTweetsProductPipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/HomeRecommendedTweetsProductPipelineConfig.scala new file mode 100644 index 000000000..06a358249 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/HomeRecommendedTweetsProductPipelineConfig.scala @@ -0,0 +1,100 @@ +package com.twitter.tweet_mixer.product.home_recommended_tweets + +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier +import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.Product +import com.twitter.product_mixer.core.pipeline.PipelineConfig +import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig +import com.twitter.product_mixer.core.product.ProductParamConfig +import com.twitter.timelines.configapi.Params +import com.twitter.tweet_mixer.feature.PredictionRequestIdFeature +import com.twitter.tweet_mixer.model.request.HomeRecommendedTweetsProduct +import com.twitter.tweet_mixer.model.request.TweetMixerRequest +import com.twitter.tweet_mixer.model.response.TweetMixerResponse +import com.twitter.tweet_mixer.product.home_recommended_tweets.model.request.HomeRecommendedTweetsProductContext +import com.twitter.tweet_mixer.product.home_recommended_tweets.model.request.HomeRecommendedTweetsQuery +import com.twitter.tweet_mixer.product.home_recommended_tweets.param.HomeRecommendedTweetsParamConfig +import com.twitter.tweet_mixer.service.TweetMixerAccessPolicy.DefaultTweetMixerAccessPolicy +import com.twitter.tweet_mixer.service.TweetMixerAccessPolicy.HomeDebugAccessPolicy +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.AllHours +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultLatencyAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HomeRecommendedTweetsProductPipelineConfig @Inject() ( + homeRecommendedTweetsParamConfig: HomeRecommendedTweetsParamConfig, + homeRecommendedTweetsRecommendationPipelineConfig: HomeRecommendedTweetsRecommendationPipelineConfig) + extends ProductPipelineConfig[ + TweetMixerRequest, + HomeRecommendedTweetsQuery, + TweetMixerResponse + ] { + + private val ProductPipelineName = "HomeRecommendedTweets" + + override val identifier: ProductPipelineIdentifier = + ProductPipelineIdentifier(ProductPipelineName) + + override val product: Product = HomeRecommendedTweetsProduct + + override val paramConfig: ProductParamConfig = homeRecommendedTweetsParamConfig + + override def pipelineQueryTransformer( + request: TweetMixerRequest, + params: Params + ): HomeRecommendedTweetsQuery = { + val (excludedIds, getRandomTweets, predictionRequestId) = request.productContext match { + case Some( + HomeRecommendedTweetsProductContext( + excludedIds, + getRandomTweets, + predictionRequestId + ) + ) => + (excludedIds, getRandomTweets, predictionRequestId) + case _ => (Set.empty[Long], false, None) + } + + val featureMap = FeatureMapBuilder() + .add(PredictionRequestIdFeature, predictionRequestId) + .build() + + HomeRecommendedTweetsQuery( + params = params, + clientContext = request.clientContext, + excludedIds = excludedIds, + requestedMaxResults = request.maxResults, + features = Some(featureMap), + debugOptions = request.debugParams.flatMap(_.debugOptions), + getRandomTweets = getRandomTweets + ) + } + + implicit val notificationGroupMap: Map[String, NotificationGroup] = ForYouGroupMap + + override val pipelines: Seq[PipelineConfig] = + Seq(homeRecommendedTweetsRecommendationPipelineConfig) + + override def pipelineSelector(query: HomeRecommendedTweetsQuery): ComponentIdentifier = + homeRecommendedTweetsRecommendationPipelineConfig.identifier + + override val debugAccessPolicies: Set[AccessPolicy] = + DefaultTweetMixerAccessPolicy ++ HomeDebugAccessPolicy + + override val alerts = Seq( + defaultSuccessRateAlert(), + defaultEmptyResponseRateAlert( + warnThreshold = 10, + criticalThreshold = 20, + notificationType = AllHours + ), + defaultLatencyAlert() + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/HomeRecommendedTweetsRecommendationPipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/HomeRecommendedTweetsRecommendationPipelineConfig.scala new file mode 100644 index 000000000..792bada9c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/HomeRecommendedTweetsRecommendationPipelineConfig.scala @@ -0,0 +1,435 @@ +package com.twitter.tweet_mixer.product.home_recommended_tweets + +import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.location.UserLocationQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.selector.Bucketer +import com.twitter.product_mixer.component_library.selector.DropAllCandidates +import com.twitter.product_mixer.component_library.selector.DropDuplicateCandidates +import com.twitter.product_mixer.component_library.selector.DropDuplicateResults +import com.twitter.product_mixer.component_library.selector.DropMaxResults +import com.twitter.product_mixer.component_library.selector.DropRequestedMaxResults +import com.twitter.product_mixer.component_library.selector.IdDuplicationKey +import com.twitter.product_mixer.component_library.selector.InsertAppendRatioResults +import com.twitter.product_mixer.component_library.selector.InsertAppendResults +import com.twitter.product_mixer.component_library.selector.SelectConditionally +import com.twitter.product_mixer.core.functional_component.common.AllPipelines +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller +import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier +import com.twitter.product_mixer.core.pipeline.FailOpenPolicy +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineConfig +import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineConfig.scoringPipelinesStep +import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig +import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} +import com.twitter.tweet_mixer.candidate_pipeline.CuratedUserTlsPerLanguageCandidatePipelineConfigFactory +import com.twitter.tweet_mixer.candidate_pipeline._ +import com.twitter.tweet_mixer.feature.AuthorIdFeature +import com.twitter.tweet_mixer.feature.USSFeatures +import com.twitter.tweet_mixer.functional_component.TweetMixerFunctionalComponents +import com.twitter.tweet_mixer.functional_component.filter.ImpressedTweetsBloomFilter +import com.twitter.tweet_mixer.functional_component.filter.IsVideoTweetFilter +import com.twitter.tweet_mixer.functional_component.filter.MaxViewCountFilter +import com.twitter.tweet_mixer.functional_component.filter.GrokFilter +import com.twitter.tweet_mixer.functional_component.filter.MinScoreFilter +import com.twitter.tweet_mixer.functional_component.gate.AllowLowSignalUserGate +import com.twitter.tweet_mixer.functional_component.hydrator.GizmoduckQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.hydrator.HaploliteQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.hydrator.HydraRankingPreparationQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.hydrator.ImpressionBloomFilterQueryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.hydrator.LastNonPollingTimeQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.hydrator.RealGraphInNetworkScoresQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.hydrator.SGSFollowedUsersQueryFeatureHydrator +import com.twitter.tweet_mixer.functional_component.hydrator.USSDeepRetrievalTweetEmbeddingFeatureHydrator +import com.twitter.tweet_mixer.functional_component.hydrator.USSGrokCategoryFeatureHydratorFactory +import com.twitter.tweet_mixer.functional_component.side_effect.DeepRetrievalAdHocSideEffect +import com.twitter.tweet_mixer.functional_component.side_effect.EvergreenVideosSideEffect +import com.twitter.tweet_mixer.functional_component.side_effect.HydraScoringSideEffect +import com.twitter.tweet_mixer.functional_component.side_effect.PublishGroxUserInterestsSideEffect +import com.twitter.tweet_mixer.functional_component.side_effect.RequestMultimodalEmbeddingSideEffect +import com.twitter.tweet_mixer.functional_component.side_effect.ScribeServedCandidatesSideEffectFactory +import com.twitter.tweet_mixer.functional_component.side_effect.SelectedStatsSideEffect +import com.twitter.tweet_mixer.marshaller.response.TweetMixerResponseTransportMarshaller +import com.twitter.tweet_mixer.model.response.TweetMixerResponse +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.ContentExplorationNonVideoTweetFeaturesEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalFilterOldSignalsEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.DeepRetrievalNonVideoTweetFeaturesEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LastNonPollingTimeFeatureHydratorEnabled +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LowSignalUserBackfillRatioParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LowSignalUserGrokTopicRatioParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LowSignalUserPopGeoRatioParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.LowSignalUserSimclustersRatioParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.UVGHighQualityTweetBasedExpansionEnabled +import com.twitter.tweet_mixer.param.UserLocationParams.UserLocationEnabled +import com.twitter.tweet_mixer.product.home_recommended_tweets.marshaller.response.HomeRecommendedTweetsDomainResponseMarshaller +import com.twitter.tweet_mixer.product.home_recommended_tweets.model.request.HomeRecommendedTweetsQuery +import com.twitter.tweet_mixer.product.home_recommended_tweets.param.HomeRecommendedTweetsParam._ +import com.twitter.tweet_mixer.scorer.HydraScorer +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.ForYouGroupMap +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultEmptyResponseRateAlert +import com.twitter.tweet_mixer.service.TweetMixerNotificationConfig.defaultSuccessRateAlert +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants.ContentExplorationEvergreenDRTweetTweet +import com.twitter.tweet_mixer.utils.CandidatePipelineConstants._ +import com.twitter.tweet_mixer.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HomeRecommendedTweetsRecommendationPipelineConfig @Inject() ( + // Query Feature Hydrators + gizmoduckQueryFeatureHydrator: GizmoduckQueryFeatureHydrator, + haploliteQueryFeatureHydrator: HaploliteQueryFeatureHydrator, + hydraRankingPreparationQueryFeatureHydrator: HydraRankingPreparationQueryFeatureHydrator, + impressionBloomFilterQueryFeatureHydratorFactory: ImpressionBloomFilterQueryFeatureHydratorFactory, + followedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, + realgraphInNetworkScoresQueryFeatureHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, + userLocationQueryFeatureHydrator: UserLocationQueryFeatureHydrator, + ussGrokCategoryFeatureHydratorFactory: USSGrokCategoryFeatureHydratorFactory, + ussDeepRetrievalTweetEmbeddingFeatureHydrator: USSDeepRetrievalTweetEmbeddingFeatureHydrator, + lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, + // Candidate Pipelines + haploliteCandidatePipelineConfigFactory: HaploliteCandidatePipelineConfigFactory, + popularGeoTweetsCandidatePipelineConfigFactory: PopularGeoTweetsCandidatePipelineConfigFactory, + popGrokTopicTweetsCandidatePipelineConfigFactory: PopGrokTopicTweetsCandidatePipelineConfigFactory, + popularTopicTweetsCandidatePipelineConfigFactory: PopularTopicTweetsCandidatePipelineConfigFactory, + simclustersInterestedInCandidatePipelineConfigFactory: SimclustersInterestedInCandidatePipelineConfigFactory, + simclustersProducerBasedCandidatePipelineConfigFactory: SimclustersProducerBasedCandidatePipelineConfigFactory, + simclustersTweetBasedCandidatePipelineConfigFactory: SimclustersTweetBasedCandidatePipelineConfigFactory, + contentAnnTweetBasedCandidatePipelineConfigFactory: ContentAnnTweetBasedCandidatePipelineConfigFactory, + twHINRebuildTweetSimilarityCandidatePipelineConfigFactory: TwHINRebuildTweetSimilarityCandidatePipelineConfigFactory, + twHINTweetSimilarityCandidatePipelineConfigFactory: TwHINTweetSimilarityCandidatePipelineConfigFactory, + utegCandidatePipelineConfigFactory: UTEGCandidatePipelineConfigFactory, + utgExpansionTweetBasedCandidatePipelineConfigFactory: UTGExpansionTweetBasedCandidatePipelineConfigFactory, + utgTweetBasedCandidatePipelineConfigFactory: UTGTweetBasedCandidatePipelineConfigFactory, + uvgExpansionTweetBasedCandidatePipelineConfigFactory: UVGExpansionTweetBasedCandidatePipelineConfigFactory, + uvgTweetBasedCandidatePipelineConfigFactory: UVGTweetBasedCandidatePipelineConfigFactory, + deepRetrievalUserTweetSimilarityCandidatePipelineConfig: DeepRetrievalUserTweetSimilarityCandidatePipelineConfig, + deepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory: DeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory, + deepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfigFactory: DeepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfigFactory, + contentExplorationEmbeddingSimilarityCandidatePipelineConfigFactory: ContentExplorationEmbeddingSimilarityCandidatePipelineConfigFactory, + twhinUserTweetSimilarityCandidatePipelineConfig: TwhinUserTweetSimilarityCandidatePipelineConfig, + contentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfigFactory: ContentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfigFactory, + contentExplorationDRTweetTweetCandidatePipelineConfigFactory: ContentExplorationDRTweetTweetCandidatePipelineConfigFactory, + contentExplorationDRTweetTweetTierTwoCandidatePipelineConfigFactory: ContentExplorationDRTweetTweetTierTwoCandidatePipelineConfigFactory, + contentExplorationEvergreenDRTweetTweetCandidatePipelineConfigFactory: ContentExplorationEvergreenDRTweetTweetCandidatePipelineConfigFactory, + contentExplorationDRUserTweetCandidatePipelineConfig: ContentExplorationDRUserTweetCandidatePipelineConfig, + contentExplorationDRUserTweetTierTwoCandidatePipelineConfig: ContentExplorationDRUserTweetTierTwoCandidatePipelineConfig, + contentExplorationSimclusterColdCandidatePipelineConfig: ContentExplorationSimclusterColdCandidatePipelineConfig, + userInterestSummaryCandidatePipelineConfigFactory: UserInterestsSummaryCandidatePipelineConfigFactory, + evergreenDRUserTweetCandidatePipelineConfig: EvergreenDRUserTweetCandidatePipelineConfig, + evergreenDRCrossBorderUserTweetCandidatePipelineConfig: EvergreenDRCrossBorderUserTweetCandidatePipelineConfig, + earlybirdInNetworkCandidatePipelineConfigFactory: EarlybirdInNetworkCandidatePipelineConfigFactory, + evergreenVideosCandidatePipelineConfigFactory: EvergreenVideosCandidatePipelineConfigFactory, + curatedUserTlsPerLanguageCandidatePipelineConfigFactory: CuratedUserTlsPerLanguageCandidatePipelineConfigFactory, + qigSearchHistoryTweetsCandidatePipelineConfigFactory: QigSearchHistoryTweetsCandidatePipelineConfigFactory, + // Scoring + hydraScorer: HydraScorer, + // SideEffect + hydraScoringSideEffect: HydraScoringSideEffect, + selectedStatsSideEffect: SelectedStatsSideEffect, + evergreenVideosSideEffect: EvergreenVideosSideEffect, + deepRetrievalAdHocSideEffect: DeepRetrievalAdHocSideEffect, + scribeServedCandidatesSideEffectFactory: ScribeServedCandidatesSideEffectFactory, + publishGroxUserInterestsSideEffect: PublishGroxUserInterestsSideEffect, + requestMultimodalEmbeddingSideEffect: RequestMultimodalEmbeddingSideEffect, + // Domain Response Marshaller + homeRecommendedTweetsDomainResponseMarshaller: HomeRecommendedTweetsDomainResponseMarshaller, + // Transport Response Marshaller + tweetMixerResponseTransportMarshaller: TweetMixerResponseTransportMarshaller, + // Remaining functional components + tweetMixerFunctionalComponents: TweetMixerFunctionalComponents, +) extends RecommendationPipelineConfig[ + HomeRecommendedTweetsQuery, + TweetCandidate, + TweetMixerResponse, + t.TweetMixerRecommendationResponse + ] { + + // Keep it same as home recommended tweets domain marshaller name + private val identifierPrefix: String = "HomeRecommendedTweets" + + override val identifier: RecommendationPipelineIdentifier = + RecommendationPipelineIdentifier(identifierPrefix) + + private val impressionBloomFilterQueryFeatureHydrator = + impressionBloomFilterQueryFeatureHydratorFactory + .build[HomeRecommendedTweetsQuery](blm.SurfaceArea.HomeTimeline) + + implicit val notificationGroupMap: Map[String, NotificationGroup] = ForYouGroupMap + + override def fetchQueryFeatures: Seq[ + QueryFeatureHydrator[HomeRecommendedTweetsQuery] + ] = tweetMixerFunctionalComponents.globalQueryFeatureHydrators ++ Seq( + AsyncQueryFeatureHydrator(scoringPipelinesStep, hydraRankingPreparationQueryFeatureHydrator), + haploliteQueryFeatureHydrator, + gizmoduckQueryFeatureHydrator, + impressionBloomFilterQueryFeatureHydrator, + followedUsersQueryFeatureHydrator, + realgraphInNetworkScoresQueryFeatureHydrator, + ParamGatedQueryFeatureHydrator( + UserLocationEnabled, + userLocationQueryFeatureHydrator + ), + ParamGatedQueryFeatureHydrator( + LastNonPollingTimeFeatureHydratorEnabled, + lastNonPollingTimeQueryFeatureHydrator + ) + ) + + override def fetchQueryFeaturesPhase2: Seq[ + BaseQueryFeatureHydrator[HomeRecommendedTweetsQuery, _] + ] = Seq( + ussGrokCategoryFeatureHydratorFactory.build(tweetBasedSignalFn), + ussDeepRetrievalTweetEmbeddingFeatureHydrator, + ) + + private val tweetBasedSignalFn: PipelineQuery => Seq[Long] = { query => + USSFeatures.getSignals[Long](query, USSFeatures.TweetFeatures) + } + + private val producerBasedSignalFn: PipelineQuery => Seq[Long] = { query => + USSFeatures.getSignals[Long](query, USSFeatures.ProducerFeatures) + } + + private val contentExplorationTweetBasedSignalFn: PipelineQuery => Seq[Long] = { query => + if (query.params(ContentExplorationNonVideoTweetFeaturesEnabled)) + USSFeatures.getSignals[Long](query, USSFeatures.ExplicitEngagementTweetFeatures) + else USSFeatures.getSignals[Long](query, USSFeatures.TweetFeatures) + } + + private val deepRetrievalTweetBasedSignalFn: PipelineQuery => Seq[Long] = { query => + val filterOldSignals = query.params(DeepRetrievalFilterOldSignalsEnabled) + if (query.params(DeepRetrievalNonVideoTweetFeaturesEnabled)) { + USSFeatures.getSignals[Long]( + query, + USSFeatures.NonVideoTweetFeatures, + filterOldSignals = filterOldSignals + ) + } else { + USSFeatures + .getSignals[Long](query, USSFeatures.TweetFeatures, filterOldSignals = filterOldSignals) + } + } + + private val searcherRealtimeHistorySignalFn: PipelineQuery => Seq[String] = { query => + USSFeatures.getSignals[String](query, USSFeatures.SearcherRealtimeHistoryFeatures) + } + + private val highQualityTweetBasedSignalFn: PipelineQuery => Seq[Long] = { query => + USSFeatures.getSignals[Long]( + query, + if (query.params(UVGHighQualityTweetBasedExpansionEnabled)) { + USSFeatures.HighQualityTweetFeatures + } else { Set.empty }) + } + + val popularGeoCandidatePipelineIdentifier = + popularGeoTweetsCandidatePipelineConfigFactory.build(identifierPrefix).identifier + val popGrokTopicTweetsCandidatePipelineIdentifier = + popGrokTopicTweetsCandidatePipelineConfigFactory.build(identifierPrefix).identifier + val popularTopicCandidatePipelineIdentifier = + popularTopicTweetsCandidatePipelineConfigFactory.build(identifierPrefix).identifier + val utegCandidatePipelineIdentifier = + utegCandidatePipelineConfigFactory.build(identifierPrefix).identifier + val simclustersProducerCandidatePipelineIdentifier = + simclustersProducerBasedCandidatePipelineConfigFactory + .build(identifierPrefix, producerBasedSignalFn).identifier + val simclustersInterestedInCandidatePipelineIdentifier = + simclustersInterestedInCandidatePipelineConfigFactory + .build(identifierPrefix).identifier + + private val contentExplorationPipelineIds = Set( + ContentExplorationEmbeddingSimilarity, + ContentExplorationEmbeddingSimilarityTierTwo, + ContentExplorationDRTweetTweet, + ContentExplorationDRTweetTweetTierTwo, + ContentExplorationDRUserTweet, + ContentExplorationDRUserTweetTierTwo, + ContentExplorationEvergreenDRTweetTweet + ).map(pipeline => CandidatePipelineIdentifier(identifierPrefix + pipeline)) + + private val deepRetrievalTweetTweetPipelineIds = Set( + DeepRetrievalTweetTweetSimilarity, + DeepRetrievalTweetTweetEmbeddingSimilarity + ).map(pipeline => CandidatePipelineIdentifier(identifierPrefix + pipeline)) + + private val uvgPipelineId = CandidatePipelineIdentifier(identifierPrefix + UVGTweetBased) + private val uvgExpansionPipelineId = CandidatePipelineIdentifier( + identifierPrefix + UVGExpansionTweetBased) + + override def candidatePipelines: Seq[ + CandidatePipelineConfig[HomeRecommendedTweetsQuery, _, _, _] + ] = Seq( + earlybirdInNetworkCandidatePipelineConfigFactory.build(identifierPrefix), + haploliteCandidatePipelineConfigFactory.build(identifierPrefix), + utegCandidatePipelineConfigFactory.build(identifierPrefix), + contentAnnTweetBasedCandidatePipelineConfigFactory + .build(identifierPrefix, signalsFn = tweetBasedSignalFn), + simclustersTweetBasedCandidatePipelineConfigFactory.build( + identifierPrefix, + signalsFn = tweetBasedSignalFn, + Seq(defaultSuccessRateAlert(), defaultEmptyResponseRateAlert()) + ), + simclustersProducerBasedCandidatePipelineConfigFactory.build( + identifierPrefix, + producerBasedSignalFn, + Seq(defaultSuccessRateAlert(), defaultEmptyResponseRateAlert()) + ), + uvgTweetBasedCandidatePipelineConfigFactory + .build(identifierPrefix, signalsFn = tweetBasedSignalFn), + uvgExpansionTweetBasedCandidatePipelineConfigFactory + .build(identifierPrefix, signalFn = highQualityTweetBasedSignalFn), + utgTweetBasedCandidatePipelineConfigFactory + .build(identifierPrefix, signalsFn = tweetBasedSignalFn), + utgExpansionTweetBasedCandidatePipelineConfigFactory + .build(identifierPrefix, signalsFn = tweetBasedSignalFn), + twHINRebuildTweetSimilarityCandidatePipelineConfigFactory + .build(identifierPrefix, tweetBasedSignalFn), + twHINTweetSimilarityCandidatePipelineConfigFactory + .build(identifierPrefix, tweetBasedSignalFn), + simclustersInterestedInCandidatePipelineConfigFactory + .build(identifierPrefix), + deepRetrievalUserTweetSimilarityCandidatePipelineConfig, + deepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory + .build(identifierPrefix, deepRetrievalTweetBasedSignalFn), + deepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfigFactory + .build(identifierPrefix, tweetBasedSignalFn), + twhinUserTweetSimilarityCandidatePipelineConfig, + contentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfigFactory + .build(identifierPrefix, contentExplorationTweetBasedSignalFn), + contentExplorationEmbeddingSimilarityCandidatePipelineConfigFactory + .build(identifierPrefix, contentExplorationTweetBasedSignalFn), + contentExplorationDRTweetTweetTierTwoCandidatePipelineConfigFactory + .build(identifierPrefix, contentExplorationTweetBasedSignalFn), + contentExplorationDRTweetTweetCandidatePipelineConfigFactory + .build(identifierPrefix, contentExplorationTweetBasedSignalFn), + contentExplorationEvergreenDRTweetTweetCandidatePipelineConfigFactory + .build(identifierPrefix, contentExplorationTweetBasedSignalFn), + contentExplorationDRUserTweetTierTwoCandidatePipelineConfig, + contentExplorationSimclusterColdCandidatePipelineConfig, + contentExplorationDRUserTweetCandidatePipelineConfig, + evergreenDRUserTweetCandidatePipelineConfig, + evergreenDRCrossBorderUserTweetCandidatePipelineConfig, + userInterestSummaryCandidatePipelineConfigFactory.build(identifierPrefix), + popularGeoTweetsCandidatePipelineConfigFactory.build(identifierPrefix), + popGrokTopicTweetsCandidatePipelineConfigFactory.build(identifierPrefix), + popularTopicTweetsCandidatePipelineConfigFactory.build(identifierPrefix), + evergreenVideosCandidatePipelineConfigFactory.build(identifierPrefix), + qigSearchHistoryTweetsCandidatePipelineConfigFactory + .build(identifierPrefix, searcherRealtimeHistorySignalFn), + curatedUserTlsPerLanguageCandidatePipelineConfigFactory.build(identifierPrefix) + ) + + override def candidatePipelineFailOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = + candidatePipelines.map { candidatePipeline => + (candidatePipeline.identifier, FailOpenPolicy.Always) + }.toMap + + override def postCandidatePipelinesSelectors: Seq[Selector[HomeRecommendedTweetsQuery]] = Seq( + InsertAppendResults(AllPipelines), + DropDuplicateResults(duplicationKey = IdDuplicationKey), + ) + + override def postCandidatePipelinesFeatureHydration: Seq[ + BaseCandidateFeatureHydrator[HomeRecommendedTweetsQuery, TweetCandidate, _] + ] = tweetMixerFunctionalComponents.globalCandidateFeatureHydrators + + override def globalFilters: Seq[Filter[HomeRecommendedTweetsQuery, TweetCandidate]] = + tweetMixerFunctionalComponents.globalFilters() ++ Seq( + ImpressedTweetsBloomFilter, + IsVideoTweetFilter(Some(Set(uvgPipelineId, uvgExpansionPipelineId))), + MinScoreFilter(candidatePipelinesToInclude = Some(deepRetrievalTweetTweetPipelineIds)), + MaxViewCountFilter(candidatePipelinesToInclude = Some(contentExplorationPipelineIds)), + GrokFilter + ) + + override def scoringPipelines: Seq[ + ScoringPipelineConfig[HomeRecommendedTweetsQuery, TweetCandidate] + ] = Seq.empty + + // Weave candidates together from non-personalized sources + private val lowSignalUserInsertSelectors = SelectConditionally( + Seq( + DropDuplicateCandidates(duplicationKey = _.features.getOrElse(AuthorIdFeature, None)), + InsertAppendRatioResults( + candidatePipelines.map(_.identifier).toSet, + bucketer = Bucketer.ByCandidateSource, + ratios = Map( + popGrokTopicTweetsCandidatePipelineIdentifier -> LowSignalUserGrokTopicRatioParam, + simclustersInterestedInCandidatePipelineIdentifier -> LowSignalUserSimclustersRatioParam, + popularGeoCandidatePipelineIdentifier -> LowSignalUserPopGeoRatioParam + ) ++ candidatePipelines + .filterNot { candidatePipeline => + candidatePipeline.identifier == popGrokTopicTweetsCandidatePipelineIdentifier || + candidatePipeline.identifier == simclustersInterestedInCandidatePipelineIdentifier || + candidatePipeline.identifier == popularGeoCandidatePipelineIdentifier + } + .map(_.identifier -> LowSignalUserBackfillRatioParam).toMap + ), + DropAllCandidates() // Drop all remaining candidates to prevent other selectors from inserting them + ), + (query: HomeRecommendedTweetsQuery, _, _) => AllowLowSignalUserGate.evaluate(query) + ) + + private val userToPostPipelines = Set( + deepRetrievalUserTweetSimilarityCandidatePipelineConfig.identifier, + contentExplorationDRUserTweetCandidatePipelineConfig.identifier, + contentExplorationDRUserTweetTierTwoCandidatePipelineConfig.identifier, + contentExplorationSimclusterColdCandidatePipelineConfig.identifier, + evergreenDRUserTweetCandidatePipelineConfig.identifier, + evergreenDRCrossBorderUserTweetCandidatePipelineConfig.identifier, + twhinUserTweetSimilarityCandidatePipelineConfig.identifier + ) + + override def resultSelectors: Seq[Selector[HomeRecommendedTweetsQuery]] = + lowSignalUserInsertSelectors ++ Seq( + // Only one of the next three selectors will be used depending on blending algorithm + tweetMixerFunctionalComponents + .exclusiveRoundRobinSelector(pipelinesToExclude = userToPostPipelines), + tweetMixerFunctionalComponents + .signalPrioritySelector(pipelinesToExclude = userToPostPipelines), + tweetMixerFunctionalComponents.weightedSignalPrioritySelector(pipelinesToExclude = + userToPostPipelines), + DropMaxResults(maxResultsParam = LightRankerMaxResultsParam), + tweetMixerFunctionalComponents + .inclusiveRoundRobinSelector(pipelinesToInclude = userToPostPipelines), + DropDuplicateResults(duplicationKey = IdDuplicationKey), + DropRequestedMaxResults( + defaultRequestedMaxResultsParam = DefaultRequestedMaxResultsParam, + serverMaxResultsParam = ServerMaxResultsParam + ) + ) + + override def resultSideEffects: Seq[ + PipelineResultSideEffect[HomeRecommendedTweetsQuery, TweetMixerResponse] + ] = Seq( + hydraScoringSideEffect, + selectedStatsSideEffect, + evergreenVideosSideEffect, + deepRetrievalAdHocSideEffect, + scribeServedCandidatesSideEffectFactory.build(identifierPrefix), + publishGroxUserInterestsSideEffect, + requestMultimodalEmbeddingSideEffect + ) + + override def domainMarshaller: DomainMarshaller[ + HomeRecommendedTweetsQuery, + TweetMixerResponse + ] = homeRecommendedTweetsDomainResponseMarshaller + + override def transportMarshaller: TransportMarshaller[ + TweetMixerResponse, + t.TweetMixerRecommendationResponse + ] = tweetMixerResponseTransportMarshaller +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/request/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/request/BUILD.bazel new file mode 100644 index 000000000..ed11f3745 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/request/BUILD.bazel @@ -0,0 +1,14 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/javax/inject:javax.inject", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], + exports = [ + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/request/HomeRecommendedTweetsProductContextUnmarshaller.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/request/HomeRecommendedTweetsProductContextUnmarshaller.scala new file mode 100644 index 000000000..238a76849 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/request/HomeRecommendedTweetsProductContextUnmarshaller.scala @@ -0,0 +1,21 @@ +package com.twitter.tweet_mixer.product.home_recommended_tweets.marshaller.request + +import com.twitter.tweet_mixer.product.home_recommended_tweets.model.request.HomeRecommendedTweetsProductContext +import com.twitter.tweet_mixer.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HomeRecommendedTweetsProductContextUnmarshaller @Inject() () { + + def apply( + productContext: t.HomeRecommendedTweetsProductContext + ): HomeRecommendedTweetsProductContext = + HomeRecommendedTweetsProductContext( + excludedIds = productContext.excludedTweetIds + .map(_.toSet) + .getOrElse(Set.empty), + getRandomTweets = productContext.getRandomTweets.getOrElse(false), + predictionRequestId = productContext.predictionRequestId + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/BUILD.bazel new file mode 100644 index 000000000..5fedcbff2 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/BUILD.bazel @@ -0,0 +1,22 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/functional_component/hydrator", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/marshaller/response/common", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + ], + exports = [ + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/HomeRecommendedTweetsDomainResponseMarshaller.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/HomeRecommendedTweetsDomainResponseMarshaller.scala new file mode 100644 index 000000000..f99c72994 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/HomeRecommendedTweetsDomainResponseMarshaller.scala @@ -0,0 +1,77 @@ +package com.twitter.tweet_mixer.product.home_recommended_tweets.marshaller.response + +import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller +import com.twitter.product_mixer.core.model.common.identifier.DomainMarshallerIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.tweet_mixer.feature.InReplyToTweetIdFeature +import com.twitter.tweet_mixer.feature.ScoreFeature +import com.twitter.tweet_mixer.feature.SignalInfo +import com.twitter.tweet_mixer.feature.AuthorIdFeature +import com.twitter.tweet_mixer.feature.SourceSignalFeature +import com.twitter.tweet_mixer.functional_component.hydrator.SignalInfoFeature +import com.twitter.tweet_mixer.model.response.TweetMixerResponse +import com.twitter.tweet_mixer.model.response.TweetResult +import com.twitter.tweet_mixer.product.home_recommended_tweets.model.request.HomeRecommendedTweetsQuery +import com.twitter.tweet_mixer.product.home_recommended_tweets.model.response.HomeRecommendedTweetsProductResponse +import com.twitter.tweet_mixer.product.home_recommended_tweets.model.response.HomeRecommendedTweetsResult +import com.twitter.tweet_mixer.utils.CandidateSourceUtil +import com.twitter.tweet_mixer.{thriftscala => t} +import com.twitter.usersignalservice.thriftscala.SignalType +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HomeRecommendedTweetsDomainResponseMarshaller @Inject() () + extends DomainMarshaller[HomeRecommendedTweetsQuery, TweetMixerResponse] { + //Keep it same as home recommended tweets recommendation pipeline name + override val identifier: DomainMarshallerIdentifier = + DomainMarshallerIdentifier("HomeRecommendedTweets") + + private def toMetricTag(info: SignalInfo): Option[t.MetricTag] = { + info.signalType match { + case SignalType.TweetFavorite => Some(t.MetricTag.TweetFavorite) + case SignalType.Retweet => Some(t.MetricTag.Retweet) + case _ => None + } + } + + override def apply( + query: HomeRecommendedTweetsQuery, + selections: Seq[CandidateWithDetails] + ): TweetMixerResponse = { + val response = selections.map { candidateWithDetails: CandidateWithDetails => + val candidateSourceId: String = candidateWithDetails.source.name + val signalBasedMetricTag = candidateWithDetails.features + .getOrElse(SignalInfoFeature, Seq.empty).flatMap(toMetricTag) + val sourceSignal = candidateWithDetails.features.getOrElse(SourceSignalFeature, 0L) + val sourceSignalTypes = candidateWithDetails.features + .getOrElse(SignalInfoFeature, Seq.empty).map(_.signalType) + val sourceSignalEntity = candidateWithDetails.features + .getOrElse(SignalInfoFeature, Seq.empty).map(_.signalEntity) + val signalAuthorId = + candidateWithDetails.features + .getOrElse(SignalInfoFeature, Seq.empty).flatMap(_.authorId) + val candidateSourceBasedMetricTags = CandidateSourceUtil.getMetricTag(candidateSourceId) + + val tweetMetadata = t.TweetMetadata( + sourceSignalId = Some(sourceSignal), + signalType = Some(sourceSignalTypes), + servedType = CandidateSourceUtil.getServedType(identifier.name, candidateSourceId), + signalEntity = sourceSignalEntity.headOption, + authorId = signalAuthorId.headOption, + ) + + HomeRecommendedTweetsResult( + TweetResult( + id = candidateWithDetails.candidateIdLong, + score = candidateWithDetails.features.getOrElse(ScoreFeature, 0.0), + metricTags = signalBasedMetricTag ++ candidateSourceBasedMetricTags, + metadata = Some(tweetMetadata), + inReplyToTweetId = candidateWithDetails.features.getOrElse(InReplyToTweetIdFeature, None), + authorId = candidateWithDetails.features.getOrElse(AuthorIdFeature, None) + )) + } + + TweetMixerResponse(HomeRecommendedTweetsProductResponse(response)) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/HomeRecommendedTweetsProductResponseMarshaller.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/HomeRecommendedTweetsProductResponseMarshaller.scala new file mode 100644 index 000000000..19ee6bd2b --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/marshaller/response/HomeRecommendedTweetsProductResponseMarshaller.scala @@ -0,0 +1,24 @@ +package com.twitter.tweet_mixer.product.home_recommended_tweets.marshaller.response + +import com.twitter.tweet_mixer.marshaller.response.common.TweetResultMarshaller +import com.twitter.tweet_mixer.product.home_recommended_tweets.model.response.HomeRecommendedTweetsProductResponse +import com.twitter.tweet_mixer.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HomeRecommendedTweetsProductResponseMarshaller @Inject() ( + tweetResultMarshaller: TweetResultMarshaller) { + + def apply( + homeRecommendedTweetsProductResponse: HomeRecommendedTweetsProductResponse + ): t.TweetMixerRecommendationResponse = + t.TweetMixerRecommendationResponse.HomeRecommendedTweetsProductResponse( + t.HomeRecommendedTweetsProductResponse(results = + homeRecommendedTweetsProductResponse.results.map { result => + t.HomeRecommendedTweetsResult( + tweetResult = tweetResultMarshaller(result.tweetResult) + ) + }) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/BUILD.bazel new file mode 100644 index 000000000..adc5d6b73 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/BUILD.bazel @@ -0,0 +1,14 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request", + ], + exports = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/request", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/HomeRecommendedTweetsProductContext.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/HomeRecommendedTweetsProductContext.scala new file mode 100644 index 000000000..e14ab9513 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/HomeRecommendedTweetsProductContext.scala @@ -0,0 +1,9 @@ +package com.twitter.tweet_mixer.product.home_recommended_tweets.model.request + +import com.twitter.product_mixer.core.model.marshalling.request.ProductContext + +case class HomeRecommendedTweetsProductContext( + excludedIds: Set[Long], + getRandomTweets: Boolean, + predictionRequestId: Option[Long]) + extends ProductContext diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/HomeRecommendedTweetsQuery.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/HomeRecommendedTweetsQuery.scala new file mode 100644 index 000000000..a295b8bfa --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/request/HomeRecommendedTweetsQuery.scala @@ -0,0 +1,26 @@ +package com.twitter.tweet_mixer.product.home_recommended_tweets.model.request + +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.marshalling.request.ClientContext +import com.twitter.product_mixer.core.model.marshalling.request.DebugOptions +import com.twitter.product_mixer.core.model.marshalling.request.HasExcludedIds +import com.twitter.product_mixer.core.model.marshalling.request.Product +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.Params +import com.twitter.tweet_mixer.model.request.HomeRecommendedTweetsProduct + +case class HomeRecommendedTweetsQuery( + override val params: Params, + override val clientContext: ClientContext, + override val excludedIds: Set[Long], + override val requestedMaxResults: Option[Int], + override val features: Option[FeatureMap], + override val debugOptions: Option[DebugOptions], + val getRandomTweets: Boolean) + extends PipelineQuery + with HasExcludedIds { + override val product: Product = HomeRecommendedTweetsProduct + + override def withFeatureMap(features: FeatureMap): HomeRecommendedTweetsQuery = + copy(features = Some(features)) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/BUILD.bazel new file mode 100644 index 000000000..829140225 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/BUILD.bazel @@ -0,0 +1,13 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + ], + exports = [ + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/model/response", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/HomeRecommendedTweetsProductResponse.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/HomeRecommendedTweetsProductResponse.scala new file mode 100644 index 000000000..c5c62bd72 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/HomeRecommendedTweetsProductResponse.scala @@ -0,0 +1,11 @@ +package com.twitter.tweet_mixer.product.home_recommended_tweets.model.response + +import com.twitter.tweet_mixer.model.response.TweetMixerProductResponse +import com.twitter.product_mixer.core.model.marshalling.HasLength + +case class HomeRecommendedTweetsProductResponse( + results: Seq[HomeRecommendedTweetsResult]) + extends TweetMixerProductResponse + with HasLength { + override def length: Int = results.length +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/HomeRecommendedTweetsResult.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/HomeRecommendedTweetsResult.scala new file mode 100644 index 000000000..43eaea812 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/model/response/HomeRecommendedTweetsResult.scala @@ -0,0 +1,5 @@ +package com.twitter.tweet_mixer.product.home_recommended_tweets.model.response + +import com.twitter.tweet_mixer.model.response.TweetResult + +case class HomeRecommendedTweetsResult(tweetResult: TweetResult) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/BUILD.bazel new file mode 100644 index 000000000..171525364 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/BUILD.bazel @@ -0,0 +1,11 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param/decider", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/HomeRecommendedTweetsParam.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/HomeRecommendedTweetsParam.scala new file mode 100644 index 000000000..cae279e3c --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/HomeRecommendedTweetsParam.scala @@ -0,0 +1,40 @@ +package com.twitter.tweet_mixer.product.home_recommended_tweets.param + +import com.twitter.timelines.configapi.FSBoundedParam + +object HomeRecommendedTweetsParam { + val SupportedClientFSName = "home_recommended_tweets_supported_client" + + // Cap for content before inserting deep retrieval u2i candidates + object LightRankerMaxResultsParam + extends FSBoundedParam[Int]( + name = "home_recommended_tweets_light_ranker_max_results", + default = 500, + min = 50, + max = 2000 + ) + + // Default if a client does not include the requestMaxResults parameter in the request + object DefaultRequestedMaxResultsParam + extends FSBoundedParam[Int]( + name = "home_recommended_tweets_default_requested_max_results", + default = 300, + min = 1, + max = 2000 + ) + + // Maximum number of results that can be provided by the service + object ServerMaxResultsParam + extends FSBoundedParam[Int]( + name = "home_recommended_tweets_server_max_results", + default = 500, + min = 50, + max = 2000 + ) + + val boundedIntFSOverrides = Seq( + DefaultRequestedMaxResultsParam, + ServerMaxResultsParam, + LightRankerMaxResultsParam + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/HomeRecommendedTweetsParamConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/HomeRecommendedTweetsParamConfig.scala new file mode 100644 index 000000000..cf30d5b33 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param/HomeRecommendedTweetsParamConfig.scala @@ -0,0 +1,16 @@ +package com.twitter.tweet_mixer.product.home_recommended_tweets.param + +import com.twitter.product_mixer.core.product.ProductParamConfig +import com.twitter.servo.decider.DeciderKeyName +import com.twitter.tweet_mixer.param.decider.DeciderKey.EnableHomeRecommendedTweetsProduct +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HomeRecommendedTweetsParamConfig @Inject() () extends ProductParamConfig { + override val enabledDeciderKey: DeciderKeyName = EnableHomeRecommendedTweetsProduct + + override val supportedClientFSName: String = HomeRecommendedTweetsParam.SupportedClientFSName + + override val boundedIntFSOverrides = HomeRecommendedTweetsParam.boundedIntFSOverrides +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scorer/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scorer/BUILD.bazel new file mode 100644 index 000000000..2b79f56d8 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scorer/BUILD.bazel @@ -0,0 +1,10 @@ +scala_library( + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "hydra/root/thrift/src/main/thrift:thrift-scala", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scorer/HydraScorer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scorer/HydraScorer.scala new file mode 100644 index 000000000..441eac780 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scorer/HydraScorer.scala @@ -0,0 +1,95 @@ +package com.twitter.tweet_mixer.scorer + +import com.twitter.hydra.root.{thriftscala => t} +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.thriftscala.ClientContext +import com.twitter.stitch.Stitch +import com.twitter.tweet_mixer.feature.AuthorIdFeature +import com.twitter.tweet_mixer.feature.HydraScoreFeature +import com.twitter.tweet_mixer.feature.SourceTweetIdFeature +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.HydraModelName +import com.twitter.util.Future +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HydraScorer @Inject() ( + hydraRootService: t.HydraRoot.MethodPerEndpoint) + extends Scorer[PipelineQuery, TweetCandidate] { + + override val identifier: ScorerIdentifier = ScorerIdentifier("Hydra") + + override val features: Set[Feature[_, _]] = Set(HydraScoreFeature) + + private val BatchSize = 1500 + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + ): Stitch[Seq[FeatureMap]] = { + val requests = hydraRequest(query, candidates) + val tweetResultsStitch: Stitch[Seq[t.TweetResult]] = + Stitch + .traverse(requests) { request => + Stitch.callFuture(getTweetResults(request)) + }.map(_.flatten) + tweetResultsStitch + .map { tweetResults => + val tweetResponseMap = tweetResults + .map(scoredTweet => (scoredTweet.tweetId, scoredTweet.scoreMap.toMap)).toMap + candidates.map { candidate => + val id = candidate.features + .getOrElse(SourceTweetIdFeature, None) + .getOrElse(candidate.candidate.id) + FeatureMap(HydraScoreFeature, tweetResponseMap.getOrElse(id, Map.empty[String, Double])) + } + } + } + + private def getTweetResults(hydraRequest: t.HydraRootRequest): Future[Seq[t.TweetResult]] = { + hydraRootService + .getRecommendationResponse(hydraRequest) + .map { + case t.HydraRootRecommendationResponse + .HydraRootTweetRankingResponse(t.HydraRootTweetRankingResponse(tweetResponse)) => + tweetResponse + }.handle { + case _ => Seq.empty + } + } + + private def hydraRequest( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]], + batchSize: Int = BatchSize + ): Seq[t.HydraRootRequest] = { + val tweetCandidatesIter = candidates + .map { candidate => + t.TweetCandidate( + id = candidate.features + .getOrElse(SourceTweetIdFeature, None) + .getOrElse(candidate.candidate.id), + authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(-1) + ) + }.distinct.grouped(batchSize).toSeq + + tweetCandidatesIter.map { tweetCandidates => + t.HydraRootRequest( + clientContext = ClientContext(userId = query.clientContext.userId), + product = t.Product.TweetRanking, + productContext = Some( + t.ProductContext.TweetRanking( + t.TweetRanking( + candidates = tweetCandidates, + modelName = query.params(HydraModelName) + ))), + ) + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scoring_pipeline/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scoring_pipeline/BUILD.bazel new file mode 100644 index 000000000..4984247bd --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scoring_pipeline/BUILD.bazel @@ -0,0 +1,13 @@ +scala_library( + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/candidate_pipeline", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/product/home_recommended_tweets/param", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scorer", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scoring_pipeline/HydraScoringPipelineConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scoring_pipeline/HydraScoringPipelineConfig.scala new file mode 100644 index 000000000..8e5b82992 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/scoring_pipeline/HydraScoringPipelineConfig.scala @@ -0,0 +1,29 @@ +package com.twitter.tweet_mixer.scoring_pipeline + +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.selector.InsertAppendResults +import com.twitter.product_mixer.core.functional_component.common.AllPipelines +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.tweet_mixer.param.TweetMixerGlobalParams.HydraScoringPipelineEnabled +import com.twitter.tweet_mixer.scorer.HydraScorer + +case class HydraScoringPipelineConfig( + hydraScorer: HydraScorer, + excludeDeepRetrievalCandidatePipelineIdentifiers: Set[CandidatePipelineIdentifier] = Set.empty, +) extends ScoringPipelineConfig[PipelineQuery, TweetCandidate] { + + override val supportedClientParam: Option[FSParam[Boolean]] = Some(HydraScoringPipelineEnabled) + + override val identifier: ScoringPipelineIdentifier = + ScoringPipelineIdentifier("HydraScoring") + + override val selectors: Seq[Selector[PipelineQuery]] = Seq(InsertAppendResults(AllPipelines)) + + override val scorers: Seq[Scorer[PipelineQuery, TweetCandidate]] = Seq(hydraScorer) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/BUILD.bazel new file mode 100644 index 000000000..7511e170d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/BUILD.bazel @@ -0,0 +1,11 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "configapi/configapi-core", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerAccessPolicy.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerAccessPolicy.scala new file mode 100644 index 000000000..1ddec10f5 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerAccessPolicy.scala @@ -0,0 +1,16 @@ +package com.twitter.tweet_mixer.service + +import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy +import com.twitter.product_mixer.core.functional_component.common.access_policy.AllowedLdapGroups + +object TweetMixerAccessPolicy { + + val DefaultTweetMixerAccessPolicy: Set[AccessPolicy] = + Set(AllowedLdapGroups(Set.empty[String])) + + val HomeDebugAccessPolicy: Set[AccessPolicy] = + Set(AllowedLdapGroups(Set.empty[String])) + + val ExploreDebugAccessPolicy: Set[AccessPolicy] = + Set(AllowedLdapGroups(Set.empty[String])) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerNotificationConfig.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerNotificationConfig.scala new file mode 100644 index 000000000..951a40e02 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerNotificationConfig.scala @@ -0,0 +1,93 @@ +package com.twitter.tweet_mixer.service + +import com.twitter.conversions.DurationOps._ +import com.twitter.product_mixer.core.functional_component.common.alert.Destination +import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert +import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert +import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup +import com.twitter.product_mixer.core.functional_component.common.alert.P99 +import com.twitter.product_mixer.core.functional_component.common.alert.Percentile +import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert +import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove +import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow +import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove +import com.twitter.util.Duration + +/** + * Notifications (email, pagerduty, etc) can be specific per-alert but it is common for multiple + * products to share notification configuration. + */ +object TweetMixerNotificationConfig { + + val AllHours = "AllHours" + val BusinessHours = "BusinessHours" + val CoreProductAllHoursNotificationGroup: NotificationGroup = NotificationGroup( + warn = Destination(emails = Seq("")), + critical = Destination(emails = Seq("")), + ) + val CoreProductBusinessHoursNotificationGroup: NotificationGroup = NotificationGroup( + warn = Destination(emails = Seq("")), + critical = Destination(emails = Seq("")), + ) + + val ForYouAllHoursNotificationGroup: NotificationGroup = NotificationGroup( + warn = Destination(emails = Seq("")), + critical = Destination(emails = Seq("")), + ) + val ForYouBusinessHoursNotificationGroup: NotificationGroup = NotificationGroup( + warn = Destination(emails = Seq("")), + critical = Destination(emails = Seq("")), + ) + + val CoreProductGroupMap = Map( + AllHours -> CoreProductAllHoursNotificationGroup, + BusinessHours -> CoreProductBusinessHoursNotificationGroup) + val ForYouGroupMap = Map( + AllHours -> ForYouAllHoursNotificationGroup, + BusinessHours -> ForYouBusinessHoursNotificationGroup) + + def defaultEmptyResponseRateAlert( + warnThreshold: Double = 50, + criticalThreshold: Double = 80, + notificationType: String = BusinessHours + )( + implicit notificationGroup: Map[String, NotificationGroup] + ) = + EmptyResponseRateAlert( + notificationGroup = notificationGroup.get(notificationType).get, + warnPredicate = TriggerIfAbove(warnThreshold, 30, 30), + criticalPredicate = TriggerIfAbove(criticalThreshold, 30, 30) + ) + + def defaultSuccessRateAlert( + threshold: Double = 99.8, + warnDatapointsPastThreshold: Int = 20, + criticalDatapointsPastThreshold: Int = 30, + duration: Int = 30, + notificationType: String = AllHours + )( + implicit notificationGroup: Map[String, NotificationGroup] + ) = SuccessRateAlert( + notificationGroup = notificationGroup.get(notificationType).get, + warnPredicate = TriggerIfBelow(threshold, warnDatapointsPastThreshold, duration), + criticalPredicate = TriggerIfBelow(threshold, criticalDatapointsPastThreshold, duration), + ) + + def defaultLatencyAlert( + latencyThreshold: Duration = 350.millis, + warningDatapointsPastThreshold: Int = 15, + criticalDatapointsPastThreshold: Int = 30, + duration: Int = 30, + percentile: Percentile = P99, + notificationType: String = BusinessHours + )( + implicit notificationGroup: Map[String, NotificationGroup] + ): LatencyAlert = LatencyAlert( + notificationGroup = notificationGroup.get(notificationType).get, + percentile = percentile, + warnPredicate = + TriggerIfLatencyAbove(latencyThreshold, warningDatapointsPastThreshold, duration), + criticalPredicate = + TriggerIfLatencyAbove(latencyThreshold, criticalDatapointsPastThreshold, duration) + ) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerService.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerService.scala new file mode 100644 index 000000000..f2daa341d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/service/TweetMixerService.scala @@ -0,0 +1,25 @@ +package com.twitter.tweet_mixer.service + +import com.twitter.product_mixer.core.model.marshalling.request.Request +import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest +import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry +import com.twitter.tweet_mixer.thriftscala.TweetMixerRecommendationResponse +import com.twitter.stitch.Stitch +import com.twitter.timelines.configapi.Params +import javax.inject.Inject +import javax.inject.Singleton +import scala.reflect.runtime.universe._ + +@Singleton +class TweetMixerService @Inject() (productPipelineRegistry: ProductPipelineRegistry) { + + def getTweetMixerRecommendationResponse[RequestType <: Request]( + request: RequestType, + params: Params + )( + implicit requestTypeTag: TypeTag[RequestType] + ): Stitch[TweetMixerRecommendationResponse] = + productPipelineRegistry + .getProductPipeline[RequestType, TweetMixerRecommendationResponse](request.product) + .process(ProductPipelineRequest(request, params)) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/store/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/store/BUILD.bazel new file mode 100644 index 000000000..8a2a48132 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/store/BUILD.bazel @@ -0,0 +1,29 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/com/twitter/bijection:scrooge", + "3rdparty/jvm/com/twitter/storehaus:core", + "src/thrift/com/twitter/ml/api:data-scala", + "stitch/stitch-core", + "storage/clients/manhattan/client/src/main/scala", + # For TwhinManhattanRWEmbeddingStore + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/offheap", + "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", + "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authorization", + "finatra/inject/inject-core/src/main/scala", + "finagle/finagle-memcached/src/main/scala", + "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client", + "src/scala/com/twitter/storehaus_internal/manhattan", + "src/scala/com/twitter/storehaus_internal/memcache", + "src/scala/com/twitter/storehaus_internal/memcache/config", + "src/scala/com/twitter/storehaus_internal/util", + "src/scala/com/twitter/simclusters_v2/summingbird/stores", + "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", + "user_history_transformer/thrift/src/main/thrift/com/twitter/user_history_transformer:user_history_transformer-scala", + "util-internal/util-cache/src/main/java", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/store/TwhinEmbeddingsStore.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/store/TwhinEmbeddingsStore.scala new file mode 100644 index 000000000..3667741fd --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/store/TwhinEmbeddingsStore.scala @@ -0,0 +1,139 @@ +package com.twitter.tweet_mixer.store + +import com.twitter.bijection.scrooge.BinaryScalaCodec +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.hermit.store.common.ObservedCachedReadableStore +import com.twitter.hermit.store.common.ObservedMemcachedReadableStore +import com.twitter.simclusters_v2.summingbird.stores.ManhattanFromStratoStore +import com.twitter.simclusters_v2.thriftscala.PersistentTwhinTweetEmbedding +import com.twitter.simclusters_v2.thriftscala.PersistentTwhinUserEmbedding +import com.twitter.simclusters_v2.{thriftscala => sim} +import com.twitter.scrooge.ThriftStruct +import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams +import com.twitter.storehaus.ReadableStore +import com.twitter.storehaus_internal.memcache.MemcacheStore +import com.twitter.storehaus_internal.util._ +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton() +class TwhinEmbeddingsStore @Inject() ( + statsReceiver: StatsReceiver, + serviceIdentifier: ServiceIdentifier) { + + val ManhattanNashDest = "/s/manhattan/nash.native-thrift" + val TwhinEmbeddingsProdAppId = "twhin_embeddings_prod" + val UserPositiveDataset = "twhin_user_positive_embeddings" + val RebuildUserPositiveDataset = "twhin_rebuild_user_rt_pos_emb" + val TweetDataset = "twhin_tweet_embeddings" + val TweetRebuildDataset = "twhin_rebuild_tweet_rt_emb" + val MemcacheTweetDest = "/s/cache/twhin_embeddings" + val KeyPrefixTweetRebuild = "twhin_tweets_rebuild" + val VideoDataset = "twhin_video_embeddings" + val MemcacheVideoDest = "/s/cache/twhin_video_embeddings" + val KeyPrefixVideo = "twhin_videos" + + val InMemoryCachePrefix = "in_memory_cache" + val MinEngagementCount = 16 + val IsProdEnv = serviceIdentifier.environment == "prod" + + /** + * We do not generate the tweet or video embedding if the number of recent engagements + * is < `MinEngagementCount`. This is based on prior Simcluster embedding aggregation + * experience and in order to be consistent with the Strato column + * strato/config/columns/recommendations/twhin/CachedTwhinTweetEmbeddings.Tweet.strato + */ + private def normalizeByCount( + persistentEmbedding: sim.PersistentTwhinTweetEmbedding + ): sim.TwhinTweetEmbedding = { + val embedding = persistentEmbedding.embedding.embedding + val updatedEmbedding = + if (persistentEmbedding.updatedCount < MinEngagementCount) embedding.map(_ => 0.0) + else embedding.map(_ / persistentEmbedding.updatedCount) + sim.TwhinTweetEmbedding(updatedEmbedding) + } + + private def createManhattanStore[T <: ThriftStruct: Manifest]( + dataset: String + ): ReadableStore[Long, T] = { + ManhattanFromStratoStore + .createPersistentTwhinStore[T]( + dataset = dataset, + mhMtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), + statsReceiver = statsReceiver, + appId = TwhinEmbeddingsProdAppId, + dest = ManhattanNashDest + ).composeKeyMapping((_, 0L)) + } + + private def createManhattanVersionedStore[T <: ThriftStruct: Manifest]( + dataset: String + ): ReadableStore[(Long, Long), T] = { + ManhattanFromStratoStore + .createPersistentTwhinStore[T]( + dataset = dataset, + mhMtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), + statsReceiver = statsReceiver, + appId = TwhinEmbeddingsProdAppId, + dest = ManhattanNashDest + ).composeKeyMapping[(Long, Long)](key => key) + } + + private def createCachedStore[K]( + underlyingStore: ReadableStore[K, sim.TwhinTweetEmbedding], + cacheDest: String, + keyPrefix: String, + ): ReadableStore[K, sim.TwhinTweetEmbedding] = { + val scopedStatsReceiver = statsReceiver.scope(keyPrefix) + val underlyingCacheClient = MemcacheStore.memcachedClient( + name = ClientName(keyPrefix), + dest = ZkEndPoint(cacheDest), + statsReceiver = scopedStatsReceiver, + serviceIdentifier = serviceIdentifier, + timeout = 80.milliseconds + ) + + val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient( + backingStore = underlyingStore, + cacheClient = underlyingCacheClient, + ttl = 15.minutes, + asyncUpdate = IsProdEnv + )( + valueInjection = BinaryScalaCodec(sim.TwhinTweetEmbedding), + statsReceiver = scopedStatsReceiver, + keyToString = { key: K => s"${keyPrefix}_${key}" } + ) + + ObservedCachedReadableStore.from( + memcacheStore, + ttl = 1.minutes, + maxKeys = 500000, + windowSize = 10000L, + cacheName = s"${InMemoryCachePrefix}_${keyPrefix}", + )(scopedStatsReceiver) + } + + val mhUserPositiveStore: ReadableStore[Long, sim.TwhinTweetEmbedding] = + createManhattanStore[PersistentTwhinUserEmbedding](UserPositiveDataset).mapValues(_.embedding) + + val mhTweetStore: ReadableStore[Long, sim.TwhinTweetEmbedding] = + createManhattanStore[PersistentTwhinTweetEmbedding](TweetDataset).mapValues(normalizeByCount) + + val mhTweetRebuildStore: ReadableStore[(Long, Long), sim.TwhinTweetEmbedding] = + createManhattanVersionedStore[PersistentTwhinTweetEmbedding](TweetRebuildDataset) + .mapValues(normalizeByCount) + + val cachedTweetRebuildStore = + createCachedStore[(Long, Long)](mhTweetRebuildStore, MemcacheTweetDest, KeyPrefixTweetRebuild) + + val mhRebuildUserPositiveStore: ReadableStore[(Long, Long), sim.TwhinTweetEmbedding] = + createManhattanVersionedStore[PersistentTwhinUserEmbedding](RebuildUserPositiveDataset) + .mapValues(_.embedding) + + val mhVideoStore: ReadableStore[Long, sim.TwhinTweetEmbedding] = + createManhattanStore[PersistentTwhinTweetEmbedding](VideoDataset).mapValues(normalizeByCount) + + val cachedVideoStore = createCachedStore(mhVideoStore, MemcacheVideoDest, KeyPrefixVideo) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/BUILD.bazel b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/BUILD.bazel new file mode 100644 index 000000000..af37dfbe0 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/BUILD.bazel @@ -0,0 +1,16 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "finagle/finagle-memcached/src/main/scala", + "servo/repo", + "snowflake/src/main/scala/com/twitter/snowflake/id", + "storage/clients/manhattan/client/src/main/scala", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/feature", + "tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/param", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", + "user-signal-service/thrift/src/main/thrift:thrift-scala", + ], +) diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/BucketSnowflakeIdAgeStats.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/BucketSnowflakeIdAgeStats.scala new file mode 100644 index 000000000..84bf1baf5 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/BucketSnowflakeIdAgeStats.scala @@ -0,0 +1,58 @@ +package com.twitter.tweet_mixer.utils + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.servo.util.MemoizingStatsReceiver +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.util.Time + +/** + * Record the counts by created Time in buckets for SnowflakeIds. + * For example, tweetIds, userIds which are created after 2014. + * + * Warning: Don't use more than 100 buckets in real production. + * It damages the production service performance. + * + * CQL example + * zone([atla], groupby(metric, sum(_), default(0, rate(ts(SUM, content-recommender.prod.content-recommender, members(sd.content-recommender.prod.content-recommender), simClusters_tweet_candidate_source/hourly/'*'))))) / 60 + * + * @param bucketSize Define the basic millisecond size of each bucket. + * @param fn Define the function to convert a generic type to a SnowflakeId + */ +case class BucketSnowflakeIdAgeStats[K]( + bucketSize: Long, + fn: K => Long +)( + implicit statsReceiver: StatsReceiver) { + import BucketSnowflakeIdAgeStats._ + + private val memoizeStats = new MemoizingStatsReceiver(statsReceiver) + private val bucketStats = memoizeStats.scope("bucket") + private val totalNumCounter = memoizeStats.counter("total") + + def count[KS <: Iterable[K]](ks: KS): KS = { + val now = Time.now + + ks.foreach { k => + val bucketId = SnowflakeId.timeFromIdOpt(fn(k)) match { + case Some(time) if now > time => + (now - time).inMillis / bucketSize + case None => + Invalid + } + bucketStats.counter(s"$bucketId").incr() + totalNumCounter.incr() + } + ks + } +} + +object BucketSnowflakeIdAgeStats { + + val Invalid: Long = -1 + + val MillisecondsPerMinute: Long = 1000 * 60 + val MillisecondsPerHour: Long = MillisecondsPerMinute * 60 + val MillisecondsPerDay: Long = MillisecondsPerHour * 24 + val MillisecondsPerYear: Long = MillisecondsPerDay * 365 + +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/CandidatePipelineConstants.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/CandidatePipelineConstants.scala new file mode 100644 index 000000000..ad7e969d7 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/CandidatePipelineConstants.scala @@ -0,0 +1,64 @@ +package com.twitter.tweet_mixer.utils + +object CandidatePipelineConstants { + val PopularGeoTweets = "PopularGeoTweets" + val PopGrokTopicTweets = "PopGrokTopicTweets" + val PopularTopicTweets = "PopularTopicTweets" + val CertoTopicTweets = "CertoTopicTweets" + val SkitTopicTweets = "SkitTopicTweets" + val SkitHighPrecisionTopicTweets = "SkitHighPrecisionTopicTweets" + val LocalTweets = "LocalTweets" + val SimClustersInterestedIn = "SimClustersInterestedIn" + val SimClustersProducerBased = "SimClustersProducerBased" + val SimClustersTweetBased = "SimClustersTweetBased" + val SimClustersPromotedCreator = "SimClustersPromotedCreator" + val Trends = "Trends" + val Events = "Events" + val TrendsVideo = "TrendsVideo" + val TwhinConsumerBased = "TwhinConsumerBased" + val TwhinRebuildUserTweetSimilarity = "TwhinRebuildUserTweetSimilarity" + val TwhinRebuildTweetSimilarity = "TwhinRebuildTweetSimilarity" + val TwhinTweetSimilarity = "TwhinTweetSimilarity" + val RelatedCreator = "RelatedCreator" + val RelatedNsfwCreator = "RelatedNsfwCreator" + val UserLocation = "UserLocation" + val UTEG = "UTEG" + val UTGExpansionTweetBased = "UTGExpansionTweetBased" + val UTGTweetBased = "UTGTweetBased" + val UTGProducerBased = "UTGProducerBased" + val UVGExpansionTweetBased = "UVGExpansionTweetBased" + val UVGTweetBased = "UVGTweetBased" + val EarlybirdInNetwork = "EarlybirdInNetwork" + val EarlybirdAuthorBased = "EarlybirdAuthorBased" + val EvergreenVideos = "EvergreenVideos" + val DeepRetrievalUserTweetSimilarity = "DeepRetrievalUserTweetSimilarity" + val DeepRetrievalTweetTweetSimilarity = "DeepRetrievalTweetTweetSimilarity" + val DeepRetrievalTweetTweetEmbeddingSimilarity = "DeepRetrievalTweetTweetEmbeddingSimilarity" + val ContentExplorationDRTweetTweet = "ContentExplorationDRTweetTweet" + val ContentExplorationEvergreenDRTweetTweet = "ContentExplorationEvergreenDRTweetTweet" + val ContentExplorationDRTweetTweetTierTwo = "ContentExplorationDRTweetTweetTierTwo" + val ContentExplorationDRUserTweet = "ContentExplorationDRUserTweet" + val ContentExplorationDRUserTweetTierTwo = "ContentExplorationDRUserTweetTierTwo" + val ContentExplorationEmbeddingSimilarity = "ContentExplorationEmbeddingSimilarity" + val ContentExplorationEmbeddingSimilarityTierTwo = "ContentExplorationEmbeddingSimilarityTierTwo" + val ContentExplorationSimclusterColdPosts = "ContentExplorationSimclusterColdPosts" + val UserInterestSummary = "UserInterestSummary" + val EvergreenDRUserTweet = "EvergreenDRUserTweet" + val EvergreenDRCrossBorderUserTweet = "EvergreenDRCrossBorderUserTweet" + val MediaDeepRetrievalTweetTweetSimilarity = "MediaDeepRetrievalTweetTweetSimilarity" + val MediaDeepRetrievalUserTweetSimilarity = "MediaDeepRetrievalUserTweetSimilarity" + val MediaEvergreenDeepRetrievalUserTweetSimilarity = + "MediaEvergreenDRUserTweetSimilarity" + val MediaPromotedCreatorDeepRetrievalUserTweetSimilarity = + "MediaPromotedCreatorDRUserTweetSimilarity" + val ControlAiTopicTweets = "ControlAiTopicTweets" + + val Haplolite = "Haplolite" + val QigSearchHistoryTweets = "QigSearchHistoryTweets" + val TwitterClipV0Short = "TwitterClipV0Short" + val TwitterClipV0Long = "TwitterClipV0Long" + val SemanticVideo = "SemanticVideo" + val MemeVideo = "MemeVideo" + val ContentEmbeddingAnn = "ContentEmbeddingAnn" + val CuratedUserTlsPerLanguage = "CuratedUserTlsPerLanguage" +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/CandidateSourceUtil.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/CandidateSourceUtil.scala new file mode 100644 index 000000000..b4b7f9a9f --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/CandidateSourceUtil.scala @@ -0,0 +1,103 @@ +package com.twitter.tweet_mixer.utils + +import com.twitter.tweet_mixer.thriftscala.MetricTag +import com.twitter.tweet_mixer.thriftscala.ServedType + +object CandidateSourceUtil { + def getServedType(identiferPrefix: String, candidateSourceId: String): Option[ServedType] = { + candidateSourceId.replace(identiferPrefix, "") match { + case CandidatePipelineConstants.EarlybirdInNetwork => Some(ServedType.InNetwork) + case CandidatePipelineConstants.DeepRetrievalTweetTweetSimilarity => + Some(ServedType.DeepRetrievalI2iEmb) + case CandidatePipelineConstants.DeepRetrievalUserTweetSimilarity => + Some(ServedType.DeepRetrieval) + case CandidatePipelineConstants.DeepRetrievalTweetTweetEmbeddingSimilarity => + Some(ServedType.DeepRetrievalI2iEmb) + case CandidatePipelineConstants.ContentExplorationDRTweetTweet => + Some(ServedType.ContentExplorationDRI2i) + case CandidatePipelineConstants.ContentExplorationDRTweetTweetTierTwo => + Some(ServedType.ContentExplorationDRI2iTier2) + case CandidatePipelineConstants.ContentExplorationEvergreenDRTweetTweet => + Some(ServedType.ContentExplorationEvergreenDRI2i) + case CandidatePipelineConstants.ContentExplorationDRUserTweet => + Some(ServedType.ContentExplorationDRI2i) + case CandidatePipelineConstants.ContentExplorationDRUserTweetTierTwo => + Some(ServedType.ContentExplorationDRI2iTier2) + case CandidatePipelineConstants.ContentExplorationEmbeddingSimilarity => + Some(ServedType.ContentExploration) + case CandidatePipelineConstants.ContentExplorationEmbeddingSimilarityTierTwo => + Some(ServedType.ContentExplorationTier2) + case CandidatePipelineConstants.ContentExplorationSimclusterColdPosts => + Some(ServedType.ContentExplorationSimclusterColdPosts) + case CandidatePipelineConstants.UserInterestSummary => + Some(ServedType.UserInterestSummaryI2i) + case CandidatePipelineConstants.EvergreenDRUserTweet => + Some(ServedType.EvergreenDRU2iHome) + case CandidatePipelineConstants.EvergreenDRCrossBorderUserTweet => + Some(ServedType.EvergreenDRCrossBorderU2iHome) + case CandidatePipelineConstants.Events => Some(ServedType.Trends) + case CandidatePipelineConstants.Trends => Some(ServedType.Trends) + case CandidatePipelineConstants.TrendsVideo => Some(ServedType.Trends) + case CandidatePipelineConstants.UserLocation => Some(ServedType.Local) + case CandidatePipelineConstants.UTEG => Some(ServedType.Uteg) + case CandidatePipelineConstants.UTGTweetBased => Some(ServedType.Utg) + case CandidatePipelineConstants.UTGProducerBased => Some(ServedType.Utg) + case CandidatePipelineConstants.UTGExpansionTweetBased => Some(ServedType.Utg) + case CandidatePipelineConstants.UVGTweetBased => Some(ServedType.Uvg) + case CandidatePipelineConstants.UVGExpansionTweetBased => Some(ServedType.Uvg) + case CandidatePipelineConstants.SimClustersTweetBased => Some(ServedType.Simclusters) + case CandidatePipelineConstants.SimClustersProducerBased => Some(ServedType.Simclusters) + case CandidatePipelineConstants.SimClustersInterestedIn => Some(ServedType.Simclusters) + case CandidatePipelineConstants.TwhinConsumerBased => Some(ServedType.Twhin) + case CandidatePipelineConstants.TwhinRebuildTweetSimilarity => Some(ServedType.Twhin) + case CandidatePipelineConstants.TwhinTweetSimilarity => Some(ServedType.Twhin) + case CandidatePipelineConstants.PopularGeoTweets => Some(ServedType.PopGeo) + case CandidatePipelineConstants.LocalTweets => Some(ServedType.PopGeo) + case CandidatePipelineConstants.PopularTopicTweets => Some(ServedType.PopTopic) + case CandidatePipelineConstants.CertoTopicTweets => Some(ServedType.PopTopic) + case CandidatePipelineConstants.MediaDeepRetrievalUserTweetSimilarity => + Some(ServedType.DeepRetrieval) + case CandidatePipelineConstants.MediaDeepRetrievalTweetTweetSimilarity => + Some(ServedType.DeepRetrieval) + case CandidatePipelineConstants.ContentEmbeddingAnn => + Some(ServedType.ContentAnn) + case CandidatePipelineConstants.MediaEvergreenDeepRetrievalUserTweetSimilarity => + Some(ServedType.EvergreenDeepRetrieval) + case CandidatePipelineConstants.TwitterClipV0Long => + Some(ServedType.TwitterClipV0Long) + case CandidatePipelineConstants.TwitterClipV0Short => + Some(ServedType.TwitterClipV0Short) + case CandidatePipelineConstants.SemanticVideo => + Some(ServedType.SemanticVideo) + case CandidatePipelineConstants.MemeVideo => + Some(ServedType.MemeVideo) + case CandidatePipelineConstants.RelatedCreator => + Some(ServedType.RelatedCreator) + case CandidatePipelineConstants.RelatedNsfwCreator => + Some(ServedType.NsfwVideoContent) + case CandidatePipelineConstants.SimClustersPromotedCreator => + Some(ServedType.PromotedCreator) + case CandidatePipelineConstants.MediaPromotedCreatorDeepRetrievalUserTweetSimilarity => + Some(ServedType.PromotedCreator) + case CandidatePipelineConstants.TwhinRebuildUserTweetSimilarity => + Some(ServedType.RebuildTwhin) + case _ => None + } + } + + def getMetricTag(candidateSourceId: String): Seq[MetricTag] = { + if (candidateSourceId.contains(CandidatePipelineConstants.PopularGeoTweets)) + Seq(MetricTag.PopGeo) + else if (candidateSourceId.contains(CandidatePipelineConstants.LocalTweets)) + Seq(MetricTag.PopGeo) + else if (candidateSourceId.contains(CandidatePipelineConstants.PopularTopicTweets)) + Seq(MetricTag.PopTopic) + else if (candidateSourceId.contains(CandidatePipelineConstants.CertoTopicTweets)) + Seq(MetricTag.PopTopic) + else if (candidateSourceId.contains(CandidatePipelineConstants.UserLocation)) + Seq(MetricTag.Local) + else if (candidateSourceId.contains(CandidatePipelineConstants.Trends)) + Seq(MetricTag.Trends) + else Seq.empty + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/ConcurrentMapCache.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/ConcurrentMapCache.scala new file mode 100644 index 000000000..635d994f6 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/ConcurrentMapCache.scala @@ -0,0 +1,12 @@ +package com.twitter.tweet_mixer.utils + +import java.util.concurrent.ConcurrentMap + +class ConcurrentMapCache[K, V](val underlying: ConcurrentMap[K, V]) { + + def get(key: K): Option[V] = Option(underlying.get(key)) + + def set(key: K, value: V): V = { + underlying.put(key, value) + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/InjectionTransformer.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/InjectionTransformer.scala new file mode 100644 index 000000000..39ff92d48 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/InjectionTransformer.scala @@ -0,0 +1,43 @@ +package com.twitter.tweet_mixer.utils + +import com.twitter.bijection.Injection +import com.twitter.io.Buf +import com.twitter.servo.util.Transformer +import com.twitter.storage.client.manhattan.bijections.Bijections +import com.twitter.util.Return +import com.twitter.util.Try +import java.nio.ByteBuffer + +object InjectionTransformerImplicits { + implicit class ByteArrayInjectionToByteBufferTransformer[A](baInj: Injection[A, Array[Byte]]) { + + private val bbInj: Injection[A, ByteBuffer] = baInj + .andThen(Bijections.byteArray2Buf) + .andThen(Bijections.byteBuffer2Buf.inverse) + + def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) + def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) + } + + implicit class BufInjectionToByteBufferTransformer[A](bufInj: Injection[A, Buf]) { + + private val bbInj: Injection[A, ByteBuffer] = bufInj.andThen(Bijections.byteBuffer2Buf.inverse) + private val baInj: Injection[A, Array[Byte]] = bufInj.andThen(Bijections.byteArray2Buf.inverse) + + def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) + def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) + } + + implicit class ByteBufferInjectionToByteBufferTransformer[A](bbInj: Injection[A, ByteBuffer]) { + + private val baInj: Injection[A, Array[Byte]] = bbInj.andThen(Bijections.bb2ba) + + def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) + def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) + } +} + +class InjectionTransformer[A, B](inj: Injection[A, B]) extends Transformer[A, B] { + override def to(a: A): Try[B] = Return(inj(a)) + override def from(b: B): Try[A] = Try.fromScala(inj.invert(b)) +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/MemCacheStitchClient.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/MemCacheStitchClient.scala new file mode 100644 index 000000000..a7ff39ebf --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/MemCacheStitchClient.scala @@ -0,0 +1,49 @@ +package com.twitter.tweet_mixer.utils + +import com.twitter.finagle.memcached.Client +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.io.Buf +import com.twitter.stitch.MapGroup +import com.twitter.stitch.Stitch +import com.twitter.util.Future +import com.twitter.util.Return +import com.twitter.util.Throw +import com.twitter.util.Time +import com.twitter.util.Try + +/*** + * This class provides a stitch wrapper around the finagle memcache future interface + * to avoid having to do manual batch allocation. + */ +class MemcacheStitchClient(memcacheClient: Client, statsReceiver: StatsReceiver) { + + def get(key: String): Stitch[Option[Buf]] = { + Stitch.call(key, getGroup) + } + + // The memcache client doesn't provide a batch set interface so we have to just call future. + def set(key: String, value: Buf, writeTtl: Int): Stitch[Unit] = { + Stitch.callFuture(memcacheClient.set(key, 0, Time.fromSeconds(writeTtl), value)) + } + + private val getGroup = new GetGroup + + private class GetGroup extends MapGroup[String, Option[Buf]] { + override def maxSize: Int = 50 + protected def run(keys: Seq[String]): Future[String => Try[Option[Buf]]] = { + val future = memcacheClient.getResult(keys).map { result => + val hits = result.hits.transform { (_, v) => + Return(Some(v.value)) + } + val misses = result.misses.toList.map { k => + k -> Return(Option.empty[Buf]) + }.toMap + val failures = result.failures.transform { (_, f) => + Throw[Option[Buf]](f) + } + hits ++ misses ++ failures + } + future + } + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/PipelineFailureCategories.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/PipelineFailureCategories.scala new file mode 100644 index 000000000..6e7e3f510 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/PipelineFailureCategories.scala @@ -0,0 +1,14 @@ +package com.twitter.tweet_mixer.utils + +import com.twitter.product_mixer.core.pipeline.pipeline_failure.ServerFailure + +object PipelineFailureCategories { + + object FailedEmbeddingHydrationResponse extends ServerFailure { + override val failureName: String = "FailedEmbeddingHydrationResponse" + } + + object InvalidEmbeddingHydrationResponse extends ServerFailure { + override val failureName: String = "InvalidEmbeddingHydrationResponse" + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/SignalUtils.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/SignalUtils.scala new file mode 100644 index 000000000..0953b340d --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/SignalUtils.scala @@ -0,0 +1,24 @@ +package com.twitter.tweet_mixer.utils + +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.tweet_mixer.feature.LowSignalUserFeature +import com.twitter.usersignalservice.{thriftscala => uss} + +object SignalUtils { + + val ExplicitSignals: Set[uss.SignalType] = Set( + uss.SignalType.TweetFavorite, + uss.SignalType.Retweet, + uss.SignalType.Reply, + uss.SignalType.TweetBookmarkV1, + uss.SignalType.TweetShareV1, + ) + + private val SmallFollowGraphSize = 5 + + def isLowSignalUser(query: PipelineQuery, followGraphSize: Option[Int]): Boolean = { + val smallFollowGraph = followGraphSize.exists(_ < SmallFollowGraphSize) + val lowSignal = query.features.map(_.getOrElse(LowSignalUserFeature, false)).getOrElse(false) + lowSignal && smallFollowGraph + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/Transformers.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/Transformers.scala new file mode 100644 index 000000000..e0ede4cce --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/Transformers.scala @@ -0,0 +1,123 @@ +package com.twitter.tweet_mixer.utils + +import com.google.common.primitives.Longs +import com.twitter.io.Buf +import com.twitter.servo.cache.SeqSerializer +import com.twitter.servo.cache.Serializers.ArrayByteBuf +import com.twitter.servo.util.Transformer + +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import scala.Array.emptyByteArray + +object Transformers { + + def serializeIntLongOption(value: Option[(Int, Long)]): Buf = { + val byteArray = value match { + case Some((int, long)) => + val byteStream = new ByteArrayOutputStream() + val objStream = new ObjectOutputStream(byteStream) + objStream.writeObject((int, long)) + objStream.close() + byteStream.toByteArray + case None => emptyByteArray + } + Buf.ByteArray.Shared.apply(byteArray) + } + + def serializeTpCacheOption(value: Option[(Int, Long, Long)]): Buf = { + val byteArray = value match { + case Some((int, long, mediaId)) => + val byteStream = new ByteArrayOutputStream() + val objStream = new ObjectOutputStream(byteStream) + objStream.writeObject((int, long, mediaId)) + objStream.close() + byteStream.toByteArray + case None => emptyByteArray + } + Buf.ByteArray.Shared.apply(byteArray) + } + + def serializeMediaMetadataCacheOption(value: Option[Long]): Buf = { + val byteArray = value match { + case Some(mediaClusterId) => + val byteStream = new ByteArrayOutputStream() + val objStream = new ObjectOutputStream(byteStream) + objStream.writeObject(mediaClusterId) + objStream.close() + byteStream.toByteArray + case None => emptyByteArray + } + Buf.ByteArray.Shared.apply(byteArray) + } + + def deserializeIntLongOption(buffer: Buf): Option[(Int, Long)] = { + val bytes = Buf.ByteArray.Shared.extract(buffer) + if (bytes.isEmpty) None + else { + val byteStream = new ByteArrayInputStream(bytes) + val objStream = new ObjectInputStream(byteStream) + val value = objStream.readObject().asInstanceOf[(Int, Long)] + objStream.close() + Some(value) + } + } + + def deserializeTpCacheOption(buffer: Buf): Option[(Int, Long, Long)] = { + val bytes = Buf.ByteArray.Shared.extract(buffer) + if (bytes.isEmpty) None + else { + val byteStream = new ByteArrayInputStream(bytes) + val objStream = new ObjectInputStream(byteStream) + val value = objStream.readObject().asInstanceOf[(Int, Long, Long)] + objStream.close() + Some(value) + } + } + + def deserializeMediaMetadataCacheOption(buffer: Buf): Option[Long] = { + val bytes = Buf.ByteArray.Shared.extract(buffer) + if (bytes.isEmpty) None + else { + val byteStream = new ByteArrayInputStream(bytes) + val objStream = new ObjectInputStream(byteStream) + val value = objStream.readObject().asInstanceOf[Long] + objStream.close() + Some(value) + } + } + + def longDoubleSerializer(input: (Long, Double)): Array[Byte] = { + val buffer = ByteBuffer.allocate(Longs.BYTES + java.lang.Double.BYTES) + buffer.putLong(input._1) + buffer.putDouble(input._2) + buffer.array() + } + + def longDoubleDeserializer(input: Array[Byte]): (Long, Double) = { + val buffer = ByteBuffer.wrap(input) + val longValue = buffer.getLong() + val doubleValue = buffer.getDouble() + (longValue, doubleValue) + } + + private val longDoubleTransformer = + Transformer[(Long, Double), Array[Byte]](longDoubleSerializer, longDoubleDeserializer) + + private val longDoubleSeqSerializer = new SeqSerializer(longDoubleTransformer, 16) + + val longDoubleSeqBufTransformer = longDoubleSeqSerializer.andThen(ArrayByteBuf) + + def serializeFilterBoolean(value: Boolean): Buf = { + Buf.ByteArray.Shared(Array[Byte](if (value) 0x01 else 0x00)) + } + + def deserializeFilterBoolean(buffer: Buf): Boolean = { + val bytes = Buf.ByteArray.Shared.extract(buffer) + if (bytes.isEmpty) false + else bytes(0) == 0x01 + } +} diff --git a/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/Utils.scala b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/Utils.scala new file mode 100644 index 000000000..bc4fb9387 --- /dev/null +++ b/tweet-mixer/server/src/main/scala/com/twitter/tweet_mixer/utils/Utils.scala @@ -0,0 +1,57 @@ +package com.twitter.tweet_mixer.utils + +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer +import scala.util.Random + +object Utils { + type TweetId = Long + type UserId = Long + def interleave[Candidate, Key]( + candidates: Seq[Seq[Candidate]], + keyFunction: Candidate => Key + ): Seq[Candidate] = { + + // copy candidates into a mutable map so this method is thread-safe + val candidatesPerSequence = candidates.map { tweetCandidates => + mutable.Queue() ++= tweetCandidates + } + + val seen = mutable.Set[Key]() + + val candidateSeqQueue = mutable.Queue() ++= candidatesPerSequence + + val result = ArrayBuffer[Candidate]() + + while (candidateSeqQueue.nonEmpty) { + val candidatesQueue = candidateSeqQueue.head + + if (candidatesQueue.nonEmpty) { + val candidate = candidatesQueue.dequeue() + val candidateKey = keyFunction(candidate) + if (!seen.contains(candidateKey)) { + result += candidate + seen.add(candidateKey) + candidateSeqQueue.enqueue( + candidateSeqQueue.dequeue() + ) // move this Seq to end + } + } else { + candidateSeqQueue.dequeue() //finished processing this Seq + } + } + //convert result to immutable seq implicitly + result + } + + def randomizedTTL(ttlSeconds: Int, earlyExpiration: Double = 0.2): Int = { + (ttlSeconds - ttlSeconds * Math.min(earlyExpiration, 1.0) * Random.nextDouble()).toInt + } + + def generateRandomIntBits(input: Seq[Int]): Seq[Int] = { + (1 to input.length).map { _ => + val randomFloat = Random.nextFloat() * 2 - 1 + java.lang.Float.floatToIntBits(randomFloat) + } + } +}