diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 8c49a8305a..cd68983d8d 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -156,6 +156,7 @@ import { QueueService } from './QueueService.js'; import { LoggerService } from './LoggerService.js'; import { WorldRoomService } from './WorldRoomService.js'; import { WorldRoomEntityService } from './entities/WorldRoomEntityService.js'; +import { WorldRoomMultiplayService } from './WorldRoomMultiplayService.js'; import type { Provider } from '@nestjs/common'; //#region 文字列ベースでのinjection用(循環参照対応のため) @@ -232,6 +233,7 @@ const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisti const $ReversiService: Provider = { provide: 'ReversiService', useExisting: ReversiService }; const $PageService: Provider = { provide: 'PageService', useExisting: PageService }; const $WorldRoomService: Provider = { provide: 'WorldRoomService', useExisting: WorldRoomService }; +const $WorldRoomMultiplayService: Provider = { provide: 'WorldRoomMultiplayService', useExisting: WorldRoomMultiplayService }; const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService }; const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart }; @@ -387,6 +389,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting ReversiService, PageService, WorldRoomService, + WorldRoomMultiplayService, ChartLoggerService, FederationChart, @@ -539,6 +542,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $ReversiService, $PageService, $WorldRoomService, + $WorldRoomMultiplayService, $ChartLoggerService, $FederationChart, @@ -691,6 +695,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting ReversiService, PageService, WorldRoomService, + WorldRoomMultiplayService, FederationChart, NotesChart, @@ -841,6 +846,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $ReversiService, $PageService, $WorldRoomService, + $WorldRoomMultiplayService, $FederationChart, $NotesChart, diff --git a/packages/backend/src/core/WorldRoomMultiplayService.ts b/packages/backend/src/core/WorldRoomMultiplayService.ts new file mode 100644 index 0000000000..d741e69307 --- /dev/null +++ b/packages/backend/src/core/WorldRoomMultiplayService.ts @@ -0,0 +1,97 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import { DataSource, In, Not } from 'typeorm'; +import * as Redis from 'ioredis'; +import { DI } from '@/di-symbols.js'; +import { + MiWorldRoom, +} from '@/models/_.js'; +import type { WorldRoomsRepository } from '@/models/_.js'; +import { bindThis } from '@/decorators.js'; +import { RoleService } from '@/core/RoleService.js'; +import { IdService } from '@/core/IdService.js'; +import type { MiUser } from '@/models/User.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { QueryService } from '@/core/QueryService.js'; + +type PlayerState = { + position: [number, number, number], + rotation: [number, number, number], + sit?: string; // id +}; + +@Injectable() +export class WorldRoomMultiplayService { + constructor( + @Inject(DI.db) + private db: DataSource, + + @Inject(DI.redis) + private redisClient: Redis.Redis, + + @Inject(DI.worldRoomsRepository) + private worldRoomsRepository: WorldRoomsRepository, + + private roleService: RoleService, + private queryService: QueryService, + private idService: IdService, + ) { + } + + @bindThis + public async enter(userId: MiUser['id'], roomId: MiWorldRoom['id']) { + // TODO: atomicにやる + const currentPlayers = await this.redisClient.hlen(`worldRoom:${roomId}:players`); + if (currentPlayers < 10) { + const redisPipeline = this.redisClient.pipeline(); + redisPipeline.hset(`worldRoom:${roomId}:players`, userId, 1); + redisPipeline.hexpire(`worldRoom:${roomId}:players`, 30, 'FIELDS', 1, userId); + await redisPipeline.exec(); + } else { + throw new Error('Room is full.'); + } + } + + @bindThis + public async heartbeat(userId: MiUser['id'], roomId: MiWorldRoom['id']) { + const exists = await this.redisClient.hexists(`worldRoom:${roomId}:players`, userId); + if (exists) { + await this.redisClient.hexpire(`worldRoom:${roomId}:players`, 30, 'FIELDS', 1, userId); + } else { + throw new Error('Not in room.'); + } + } + + @bindThis + public async leave(userId: MiUser['id'], roomId: MiWorldRoom['id']) { + const redisPipeline = this.redisClient.pipeline(); + redisPipeline.hdel(`worldRoom:${roomId}:players`, userId); + redisPipeline.hdel(`worldRoom:${roomId}:playerStates`, userId); + await redisPipeline.exec(); + } + + @bindThis + public async updatePlayerState(userId: MiUser['id'], roomId: MiWorldRoom['id'], state: PlayerState) { + const redisPipeline = this.redisClient.pipeline(); + redisPipeline.hset(`worldRoom:${roomId}:playerStates`, userId, JSON.stringify(state)); + redisPipeline.hexpire(`worldRoom:${roomId}:playerStates`, 30, 'FIELDS', 1, userId); + await redisPipeline.exec(); + } + + @bindThis + public async getPlayerStates(roomId: MiWorldRoom['id']): Promise> { + const entries = await this.redisClient.hgetall(`worldRoom:${roomId}:playerStates`); + return Object.fromEntries(Object.entries(entries).map(([userId, state]) => [userId, JSON.parse(state) as PlayerState])); + } + + @bindThis + public getPlayerStatesAndHeatbeat(userId: MiUser['id'], roomId: MiWorldRoom['id']): Promise> { + // TODO: atomicにやる + this.heartbeat(userId, roomId); + return this.getPlayerStates(roomId); + } +} diff --git a/packages/backend/src/server/ServerModule.ts b/packages/backend/src/server/ServerModule.ts index e228d51103..eb0d203e7d 100644 --- a/packages/backend/src/server/ServerModule.ts +++ b/packages/backend/src/server/ServerModule.ts @@ -6,6 +6,7 @@ import { Module } from '@nestjs/common'; import { EndpointsModule } from '@/server/api/EndpointsModule.js'; import { CoreModule } from '@/core/CoreModule.js'; +import MainStreamConnection from '@/server/api/stream/Connection.js'; import { ApiCallService } from './api/ApiCallService.js'; import { FileServerService } from './FileServerService.js'; import { HealthServerService } from './HealthServerService.js'; @@ -30,7 +31,6 @@ import { UrlPreviewService } from './web/UrlPreviewService.js'; import { ClientLoggerService } from './web/ClientLoggerService.js'; import { OAuth2ProviderService } from './oauth/OAuth2ProviderService.js'; -import MainStreamConnection from '@/server/api/stream/Connection.js'; import { MainChannel } from './api/stream/channels/main.js'; import { AdminChannel } from './api/stream/channels/admin.js'; import { AntennaChannel } from './api/stream/channels/antenna.js'; @@ -49,6 +49,7 @@ import { ChatUserChannel } from './api/stream/channels/chat-user.js'; import { ChatRoomChannel } from './api/stream/channels/chat-room.js'; import { ReversiChannel } from './api/stream/channels/reversi.js'; import { ReversiGameChannel } from './api/stream/channels/reversi-game.js'; +import { WorldRoomChannel } from './api/stream/channels/world-room.js'; import { NoteStreamingHidingService } from './api/stream/NoteStreamingHidingService.js'; import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.js'; @@ -99,6 +100,7 @@ import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.j QueueStatsChannel, ServerStatsChannel, UserListChannel, + WorldRoomChannel, NoteStreamingHidingService, OpenApiServerService, OAuth2ProviderService, diff --git a/packages/backend/src/server/api/stream/Connection.ts b/packages/backend/src/server/api/stream/Connection.ts index 7f15c3a2db..83f117176b 100644 --- a/packages/backend/src/server/api/stream/Connection.ts +++ b/packages/backend/src/server/api/stream/Connection.ts @@ -35,6 +35,7 @@ import { ChatUserChannel } from '@/server/api/stream/channels/chat-user.js'; import { ChatRoomChannel } from '@/server/api/stream/channels/chat-room.js'; import { ReversiChannel } from '@/server/api/stream/channels/reversi.js'; import { ReversiGameChannel } from '@/server/api/stream/channels/reversi-game.js'; +import { WorldRoomChannel } from '@/server/api/stream/channels/world-room.js'; import type { ChannelRequest } from './channel.js'; import type { ChannelConstructor } from './channel.js'; import type Channel from './channel.js'; @@ -338,6 +339,7 @@ export default class Connection { case 'chatRoom': return ChatRoomChannel; case 'reversi': return ReversiChannel; case 'reversiGame': return ReversiGameChannel; + case 'worldRoom': return WorldRoomChannel; default: throw new Error(`no such channel: ${name}`); diff --git a/packages/backend/src/server/api/stream/channels/world-room.ts b/packages/backend/src/server/api/stream/channels/world-room.ts new file mode 100644 index 0000000000..8f414c61e3 --- /dev/null +++ b/packages/backend/src/server/api/stream/channels/world-room.ts @@ -0,0 +1,97 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable, Scope } from '@nestjs/common'; +import { REQUEST } from '@nestjs/core'; +import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; +import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import type { JsonObject } from '@/misc/json-value.js'; +import { WorldRoomService } from '@/core/WorldRoomService.js'; +import { WorldRoomMultiplayService } from '@/core/WorldRoomMultiplayService.js'; +import Channel, { type ChannelRequest } from '../channel.js'; + +@Injectable({ scope: Scope.TRANSIENT }) +export class WorldRoomChannel extends Channel { + public readonly chName = 'worldRoom'; + public static shouldShare = false; + public static requireCredential = true as const; + public static kind = 'read:worldRoom'; + private roomId: string; + private intervalId: NodeJS.Timeout; + private isEntered = false; + + constructor( + @Inject(REQUEST) + request: ChannelRequest, + + private worldRoomService: WorldRoomService, + private worldRoomMultiplayService: WorldRoomMultiplayService, + ) { + super(request); + } + + @bindThis + public async init(params: JsonObject): Promise { + if (typeof params.roomId !== 'string') return false; + if (!this.user) return false; + + this.roomId = params.roomId; + + const room = await this.worldRoomService.findRoomById(this.roomId); + if (room == null) return false; + + try { + await this.enter(); + } catch (err) { + return false; + } + + //this.subscriber.on(`worldRoomStream:${this.roomId}`, this.onEvent); + + return true; + } + + @bindThis + private async enter() { + if (this.isEntered) return; + + await this.worldRoomMultiplayService.enter(this.user!.id, this.roomId); + + this.isEntered = true; + + this.send('entered', {}); + + this.intervalId = setInterval(async () => { + const states = await this.worldRoomMultiplayService.getPlayerStatesAndHeatbeat(this.user!.id, this.roomId); + // TODO: 自分自身のstateは抜く + this.send('sync', states); + }, 1000); + } + + //@bindThis + //private async onEvent(data: GlobalEvents['worldRoom']['payload']) { + // this.send(data.type, data.body); + //} + + @bindThis + public onMessage(type: string, body: any) { + switch (type) { + case 'update': + if (this.roomId && this.isEntered) { + this.worldRoomMultiplayService.updatePlayerState(this.user!.id, this.roomId, body); + } + break; + } + } + + @bindThis + public dispose() { + //this.subscriber.off(`worldRoomStream:${this.roomId}`, this.onEvent); + + clearInterval(this.intervalId); + this.worldRoomMultiplayService.leave(this.user!.id, this.roomId); + } +} diff --git a/packages/frontend-misskey-world-engine/src/PlayerContainer.ts b/packages/frontend-misskey-world-engine/src/PlayerContainer.ts new file mode 100644 index 0000000000..b7ef9a9f0e --- /dev/null +++ b/packages/frontend-misskey-world-engine/src/PlayerContainer.ts @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import * as BABYLON from '@babylonjs/core'; +import { WORLD_SCALE } from 'misskey-world/src/utility.js'; + +export type PlayerProfile = { + name: string; + avatarUrl: string; +}; + +export type PlayerState = { + position: [number, number, number], + rotation: [number, number, number], + sit?: string; // id +}; + +export class PlayerContainer { + public id: string; + private profile: PlayerProfile; + private root: BABYLON.TransformNode; + private subRoot: BABYLON.TransformNode | null = null; + private sr: BABYLON.SnapshotRenderingHelper; + private scene: BABYLON.Scene; + public registerMeshes: (meshes: BABYLON.Mesh[]) => void = () => {}; + + constructor(params: { id: string; profile: PlayerProfile; state: PlayerState | null; sr: BABYLON.SnapshotRenderingHelper; scene: BABYLON.Scene; }) { + this.id = params.id; + this.profile = params.profile; + this.sr = params.sr; + this.scene = params.scene; + + this.root = new BABYLON.TransformNode(`player:${this.id}`, params.scene); + this.root.rotationQuaternion = null; + if (params.state) this.applyState(params.state, true); + } + + public async loadAvatar() { + const filePath = '/client-assets/world/avatars/default.glb'; + const loaderResult = await BABYLON.LoadAssetContainerAsync(filePath, this.scene); + + // babylonによって自動で追加される右手系変換用ノード + const subRootMesh = loaderResult.meshes[0] as BABYLON.Mesh; + + // meshじゃなくtransform nodeにしてパフォーマンス向上 + this.subRoot = new BABYLON.TransformNode('__root__', this.scene); + this.subRoot.parent = this.root; + this.subRoot.scaling.x = -1; + this.subRoot.scaling = this.subRoot.scaling.scale(WORLD_SCALE);// cmをmに + + for (const m of subRootMesh.getChildren()) { + if (m.parent === subRootMesh) { + m.parent = this.subRoot; + } + } + + subRootMesh.dispose(); + + this.registerMeshes(this.subRoot.getChildMeshes()); + } + + public applyState(state: PlayerState, forInit = false) { + this.root.position.set(...state.position); + this.root.rotation.set(...state.rotation); + if (!forInit) { + this.sr.updateMesh(this.root.getChildMeshes()); + } + } + + public destroy() { + this.root.dispose(); + } +} diff --git a/packages/frontend-misskey-world-engine/src/room/ObjectContainer.ts b/packages/frontend-misskey-world-engine/src/room/ObjectContainer.ts index a0b21e5bab..3bee8bdfe8 100644 --- a/packages/frontend-misskey-world-engine/src/room/ObjectContainer.ts +++ b/packages/frontend-misskey-world-engine/src/room/ObjectContainer.ts @@ -151,7 +151,6 @@ export class ObjectContainer { // meshじゃなくtransform nodeにしてパフォーマンス向上 this.subRoot = new BABYLON.TransformNode('__root__', this.scene); - console.log(this.root.position); this.subRoot.parent = this.root; this.subRoot.scaling.x = -1; this.subRoot.scaling = this.subRoot.scaling.scale(WORLD_SCALE);// cmをmに diff --git a/packages/frontend-misskey-world-engine/src/room/engine.ts b/packages/frontend-misskey-world-engine/src/room/engine.ts index 62c9ce1dbc..e312aba330 100644 --- a/packages/frontend-misskey-world-engine/src/room/engine.ts +++ b/packages/frontend-misskey-world-engine/src/room/engine.ts @@ -22,6 +22,7 @@ import { TIME_MAP, getMeshesBoundingBox, Timer, getYRotationDirection, FreeCamer import { EngineBase } from '../EngineBase.js'; import { genId } from '../id.js'; import { deepClone } from '../clone.js'; +import { PlayerContainer, type PlayerProfile, type PlayerState } from '../PlayerContainer.js'; import { getObjectDef } from './object-defs.js'; import { findMaterial, GRAPHICS_QUALITY, ModelManager, SYSTEM_HEYA_MESH_NAMES, SYSTEM_MESH_NAMES } from './utility.js'; import { JapaneseEnvManager, MuseumEnvManager, SimpleEnvManager } from './env.js'; @@ -175,6 +176,10 @@ export class RoomEngine extends EngineBase<{ this.ev('changeSittingState', { isSitting: v }); } + private playerProfiles: Record = {}; + private playerStates: Record = {}; + private playerContainers: PlayerContainer[] = []; + private inited = false; constructor(roomState: RoomState, roomAttachments: RoomAttachments, options: { @@ -1435,6 +1440,46 @@ export class RoomEngine extends EngineBase<{ this.ev('playSfxUrl', { url, options }); } + public updatePlayers(profiles: Record, states: Record) { + for (const playerContainer of this.playerContainers) { + if (profiles[playerContainer.id] == null) { + this.sr.disableSnapshotRendering(); + playerContainer.destroy(); + this.sr.enableSnapshotRendering(); + } + } + this.playerContainers = this.playerContainers.filter(p => profiles[p.id] != null); + + for (const [k, v] of Object.entries(profiles)) { + const playerContainer = this.playerContainers.find(p => p.id === k); + if (playerContainer == null) { + this.sr.disableSnapshotRendering(); + const p = new PlayerContainer({ + id: k, + profile: v, + state: states[k], + scene: this.scene, + sr: this.sr, + }); + this.sr.enableSnapshotRendering(); + this.playerContainers.push(p); + } else { + if (states[k] != null) { + playerContainer.applyState(states[k]); + } + } + } + } + + public clearPlayers() { + for (const playerContainer of this.playerContainers) { + this.sr.disableSnapshotRendering(); + playerContainer.destroy(); + this.sr.enableSnapshotRendering(); + } + this.playerContainers = []; + } + public resize() { // 一旦snapshot renderingを無効にしておかないとエラーが出る // ~~...が、一旦無効にしたらしたで複数のマテリアルがそれぞれ入れ替わる(?)という謎の現象が発生するためコメントアウトしとく(エラー出てもレンダリングが止まったりするわけでもないし)~~ diff --git a/packages/frontend/assets/world/avatar/avatar.blend b/packages/frontend/assets/world/avatars/default.blend similarity index 100% rename from packages/frontend/assets/world/avatar/avatar.blend rename to packages/frontend/assets/world/avatars/default.blend diff --git a/packages/frontend/assets/world/avatar/avatar.glb b/packages/frontend/assets/world/avatars/default.glb similarity index 100% rename from packages/frontend/assets/world/avatar/avatar.glb rename to packages/frontend/assets/world/avatars/default.glb diff --git a/packages/frontend/src/pages/chat/room.vue b/packages/frontend/src/pages/chat/room.vue index a4204435b3..fcfa5a0f70 100644 --- a/packages/frontend/src/pages/chat/room.vue +++ b/packages/frontend/src/pages/chat/room.vue @@ -88,7 +88,7 @@ SPDX-License-Identifier: AGPL-3.0-only