min
0b677529e1
All checks were successful
Старые Roblox-карты (Crossroads, ROBLOX Battle, и др. из эры 2007-2010) сохранены в XML-формате (<roblox version=4>... вместо binary <roblox!...). Наш парсер падал на 'missing <roblox! magic'. Новое: - rbxl_xml_parser.py: парсит XML-формат через стандартный xml.etree. Поддерживает все типичные property-теги: string, bool, int, float, Vector3, Vector2, CoordinateFrame, Color3, Color3uint8, BrickColor, Ref, BinaryString, UDim/UDim2, PhysicalProperties, OptionalCFrame. - В _parse_property: <int name=BrickColor> заворачивается в BrickColor объект — converter ожидает .code атрибут. - Алиасы PascalCase: name→Name, size→Size, shape→Shape (старый XML использовал camelCase с маленькой первой буквой). app.py: - /analyze: авто-детект XML vs Binary по magic bytes. Если XML — используем parse_xml(), иначе старый parse(). Тест на arch1_Original_Crossroads.rbxl: 877 instances, 777 Part, 83 Model — конвертится в 777 примитивов без warnings.
340 lines
13 KiB
Python
340 lines
13 KiB
Python
"""
|
||
app.py — Flask API rbxl-importer.
|
||
|
||
Endpoints:
|
||
POST /import/rbxl/analyze
|
||
body: multipart, file=<.rbxl>
|
||
resp: {
|
||
"preview_hash": str, # для следующего шага
|
||
"report": {
|
||
"filename": str, "size_bytes": int, "version": int,
|
||
"class_count": int, "instance_count": int,
|
||
"top_classes": [{"class": str, "count": int}, ...],
|
||
"scripts_total": int,
|
||
"assets_to_download": int,
|
||
"warnings": [str, ...]
|
||
}
|
||
}
|
||
|
||
POST /import/rbxl/create
|
||
body: { "preview_hash": str, "title": str, "auth_user_id": int }
|
||
resp: { "project_id": int, "redirect": "/edit/<id>" }
|
||
— Парсит → конвертит → скачивает ассеты → создаёт запись в kubikon3d_projects.
|
||
|
||
GET /health
|
||
resp: { ok: true, version: "0.1.0" }
|
||
|
||
Безопасность: эндпоинты доступны только МИНу (user_id=1).
|
||
Проверка через X-User-Id header (NPM прокинет после JWT-проверки в user-сервисе).
|
||
"""
|
||
import os
|
||
import sys
|
||
import json
|
||
import hashlib
|
||
import tempfile
|
||
import logging
|
||
from pathlib import Path
|
||
from flask import Flask, request, jsonify
|
||
from flask_cors import CORS
|
||
import psycopg2
|
||
from psycopg2.extras import RealDictCursor
|
||
import redis
|
||
|
||
sys.path.insert(0, os.path.dirname(__file__))
|
||
|
||
from rbxl_parser import parse, class_histogram
|
||
from converter import Converter
|
||
from asset_downloader import AssetDownloader, PendingDownload
|
||
from mesh_converter import parse_roblox_mesh, build_glb, MeshParseError
|
||
|
||
logging.basicConfig(level=logging.INFO)
|
||
logger = logging.getLogger('rbxl-importer')
|
||
|
||
PG_DSN = os.environ.get(
|
||
'PG_DSN',
|
||
'host=192.168.1.152 port=25435 user=min password=5cb157970ad5e3b2952ed05decaf1bab dbname=storys_db'
|
||
)
|
||
REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379/0')
|
||
STORAGE_ROOT = os.environ.get('STORAGE_ROOT', '/opt/roblox-assets')
|
||
PUBLIC_ASSET_BASE = os.environ.get('PUBLIC_ASSET_BASE', 'https://assets.rublox.pro/roblox')
|
||
|
||
MAX_RBXL_SIZE = 50 * 1024 * 1024 # 50 MB
|
||
ALLOWED_USER_IDS = [1] # пока только МИН
|
||
|
||
app = Flask(__name__)
|
||
CORS(app, resources={r'/*': {'origins': '*'}})
|
||
|
||
# Devlog для удалённой отладки dev-сессий студии: фронт пушит сюда
|
||
# console.error/warn, failed network requests, неожиданные exceptions.
|
||
from app_devlog import devlog_bp
|
||
app.register_blueprint(devlog_bp)
|
||
|
||
try:
|
||
rds = redis.from_url(REDIS_URL, decode_responses=False)
|
||
rds.ping()
|
||
logger.info(f'Redis connected: {REDIS_URL}')
|
||
except Exception as e:
|
||
logger.warning(f'Redis NOT connected: {e}; preview cache отключён')
|
||
rds = None
|
||
|
||
|
||
def pg_conn():
|
||
return psycopg2.connect(PG_DSN)
|
||
|
||
|
||
def auth_check(req) -> int:
|
||
"""Возвращает user_id если ОК, иначе бросает RuntimeError."""
|
||
# X-User-Id выставляется upstream NPM или service-user после JWT-проверки.
|
||
# В dev можно через header X-Auth-Override (только в LAN).
|
||
user_id_str = req.headers.get('X-User-Id') or req.headers.get('X-Auth-Override')
|
||
if not user_id_str:
|
||
raise RuntimeError('No X-User-Id header')
|
||
try:
|
||
uid = int(user_id_str)
|
||
except ValueError:
|
||
raise RuntimeError(f'Bad X-User-Id: {user_id_str!r}')
|
||
if uid not in ALLOWED_USER_IDS:
|
||
raise RuntimeError(f'User {uid} not allowed (only МИН)')
|
||
return uid
|
||
|
||
|
||
@app.errorhandler(Exception)
|
||
def on_error(e):
|
||
logger.exception('handler error')
|
||
return jsonify({'error': str(e), 'type': type(e).__name__}), 500
|
||
|
||
|
||
@app.get('/health')
|
||
def health():
|
||
return jsonify({'ok': True, 'version': '0.1.0', 'service': 'rbxl-importer'})
|
||
|
||
|
||
@app.post('/import/rbxl/analyze')
|
||
def analyze():
|
||
try:
|
||
user_id = auth_check(request)
|
||
except RuntimeError as e:
|
||
return jsonify({'error': str(e)}), 403
|
||
|
||
if 'file' not in request.files:
|
||
return jsonify({'error': 'file field required'}), 400
|
||
upload = request.files['file']
|
||
blob = upload.read()
|
||
if len(blob) > MAX_RBXL_SIZE:
|
||
return jsonify({'error': f'file too large (> {MAX_RBXL_SIZE} bytes)'}), 413
|
||
# Авто-детект XML vs Binary формата.
|
||
# Бинарный: <roblox!\x89\xff\r\n\x1a\n (magic bytes).
|
||
# XML (старые карты до 2010): <roblox version="4">...
|
||
stripped = blob.lstrip()
|
||
is_binary = stripped.startswith(b'<roblox!')
|
||
is_xml = stripped.startswith(b'<roblox') and not is_binary
|
||
|
||
if not is_binary and not is_xml:
|
||
return jsonify({'error': 'not a .rbxl file (no <roblox magic)'}), 400
|
||
|
||
# Парсим
|
||
try:
|
||
if is_xml:
|
||
from rbxl_xml_parser import parse_xml
|
||
model = parse_xml(blob)
|
||
else:
|
||
model = parse(blob)
|
||
except Exception as e:
|
||
return jsonify({'error': f'parse failed: {e}'}), 422
|
||
|
||
# Конвертим (без скачки ассетов — просто узнаем сколько их)
|
||
conv = Converter(model)
|
||
project_data = conv.convert()
|
||
|
||
# Лог импорта
|
||
rbxl_sha = hashlib.sha256(blob).hexdigest()
|
||
asset_ids = list(set(conv.stats.asset_ids_needed))
|
||
|
||
# Кладём blob во временный файл + сохраняем sha256 как preview_hash
|
||
preview_hash = rbxl_sha
|
||
if rds:
|
||
rds.setex(f'rbxl:blob:{preview_hash}', 1200, blob) # 20 минут
|
||
rds.setex(f'rbxl:project:{preview_hash}', 1200,
|
||
json.dumps(project_data).encode('utf-8'))
|
||
rds.setex(f'rbxl:assets:{preview_hash}', 1200,
|
||
json.dumps(asset_ids).encode('utf-8'))
|
||
|
||
# Отчёт
|
||
hist = class_histogram(model)
|
||
top = sorted(hist.items(), key=lambda x: -x[1])[:25]
|
||
report = {
|
||
'filename': upload.filename or 'unknown.rbxl',
|
||
'size_bytes': len(blob),
|
||
'version': model.version,
|
||
'class_count': model.class_count,
|
||
'instance_count': model.instance_count,
|
||
'top_classes': [{'class': c, 'count': n} for c, n in top],
|
||
'primitives_created': conv.stats.primitives_created,
|
||
'glb_models_created': conv.stats.glb_models_created,
|
||
'scripts_total': conv.stats.scripts_collected,
|
||
'scripts_skipped': conv.stats.scripts_skipped,
|
||
'parts_dropped': conv.stats.parts_dropped,
|
||
'assets_to_download': len(asset_ids),
|
||
'warnings': conv.stats.warnings[:30],
|
||
}
|
||
|
||
# Запись в roblox_imports (status='analyzed')
|
||
try:
|
||
with pg_conn() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"INSERT INTO roblox_imports "
|
||
"(user_id, rbxl_filename, rbxl_size, rbxl_sha256, "
|
||
" instance_count, class_count, assets_total, status) "
|
||
"VALUES (%s, %s, %s, %s, %s, %s, %s, 'analyzed') "
|
||
"RETURNING id",
|
||
(user_id, report['filename'], report['size_bytes'], rbxl_sha,
|
||
report['instance_count'], report['class_count'], report['assets_to_download']),
|
||
)
|
||
import_id = cur.fetchone()[0]
|
||
conn.commit()
|
||
report['import_id'] = import_id
|
||
except Exception as e:
|
||
logger.warning(f'roblox_imports insert failed: {e}')
|
||
|
||
return jsonify({'preview_hash': preview_hash, 'report': report})
|
||
|
||
|
||
@app.post('/import/rbxl/create')
|
||
def create():
|
||
try:
|
||
user_id = auth_check(request)
|
||
except RuntimeError as e:
|
||
return jsonify({'error': str(e)}), 403
|
||
|
||
data = request.get_json(silent=True) or {}
|
||
preview_hash = data.get('preview_hash')
|
||
title = (data.get('title') or '').strip() or 'Импортировано из Roblox'
|
||
|
||
if not preview_hash:
|
||
return jsonify({'error': 'preview_hash required'}), 400
|
||
|
||
if not rds:
|
||
return jsonify({'error': 'redis unavailable, cannot retrieve preview'}), 503
|
||
|
||
blob_cached = rds.get(f'rbxl:blob:{preview_hash}')
|
||
project_cached = rds.get(f'rbxl:project:{preview_hash}')
|
||
if not project_cached:
|
||
return jsonify({'error': 'preview expired, please re-analyze'}), 410
|
||
|
||
project_data = json.loads(project_cached.decode('utf-8'))
|
||
asset_ids_raw = rds.get(f'rbxl:assets:{preview_hash}')
|
||
asset_ids = json.loads(asset_ids_raw.decode('utf-8')) if asset_ids_raw else []
|
||
|
||
# Скачиваем ассеты
|
||
dl = AssetDownloader(db_dsn=PG_DSN, storage_root=STORAGE_ROOT,
|
||
public_base=PUBLIC_ASSET_BASE)
|
||
asset_url_map = {} # rbx_id → public_url
|
||
failed = 0
|
||
for rbx_id in asset_ids:
|
||
try:
|
||
rec = dl.fetch_sync(int(rbx_id))
|
||
# Конвертим mesh → glb если это mesh
|
||
if rec.asset_kind == 'mesh' and not rec.converted_path:
|
||
try:
|
||
with open(rec.raw_path, 'rb') as f:
|
||
mesh_blob = f.read()
|
||
mesh = parse_roblox_mesh(mesh_blob)
|
||
glb = build_glb(mesh)
|
||
sha = hashlib.sha256(glb).hexdigest()
|
||
glb_dir = os.path.join(STORAGE_ROOT, 'converted', sha[:2])
|
||
os.makedirs(glb_dir, exist_ok=True)
|
||
glb_path = os.path.join(glb_dir, f'{sha}.glb')
|
||
if not os.path.exists(glb_path):
|
||
with open(glb_path, 'wb') as f:
|
||
f.write(glb)
|
||
public_glb = f'{PUBLIC_ASSET_BASE}/converted/{sha[:2]}/{sha}.glb'
|
||
asset_url_map[int(rbx_id)] = public_glb
|
||
# Обновляем БД
|
||
with pg_conn() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"UPDATE roblox_assets SET converted_path=%s, "
|
||
"converted_sha256=%s, converted_size_bytes=%s "
|
||
"WHERE rbx_asset_id=%s",
|
||
(glb_path, sha, len(glb), int(rbx_id)),
|
||
)
|
||
conn.commit()
|
||
except (MeshParseError, Exception) as me:
|
||
logger.warning(f'mesh→glb failed for {rbx_id}: {me}')
|
||
asset_url_map[int(rbx_id)] = rec.public_url
|
||
else:
|
||
asset_url_map[int(rbx_id)] = rec.public_url
|
||
except PendingDownload:
|
||
failed += 1
|
||
except Exception as e:
|
||
failed += 1
|
||
logger.warning(f'asset {rbx_id} download failed: {e}')
|
||
|
||
# Подставляем URLs в project_data
|
||
_resolve_asset_urls(project_data, asset_url_map)
|
||
|
||
# Создаём проект в kubikon3d_projects
|
||
# Используем bridge на user-service / storys API, ИЛИ напрямую в storys_db.
|
||
# Прямой INSERT — проще для MVP. id автогенерируется.
|
||
try:
|
||
with pg_conn() as conn:
|
||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||
# Возьмём следующий id из последовательности
|
||
cur.execute(
|
||
"INSERT INTO kubikon3d_projects "
|
||
"(user_id, title, description, project_data, "
|
||
" status, is_test, is_public, is_published, "
|
||
" feed_eligible, feed_blocked_forever, quality_state, "
|
||
" created_at, updated_at) "
|
||
"VALUES (%s, %s, %s, %s, "
|
||
" 'draft', true, false, false, "
|
||
" false, false, 'new', NOW(), NOW()) "
|
||
"RETURNING id",
|
||
(user_id, title,
|
||
f'Импортировано из .rbxl (rbxl-importer v0.1.0)',
|
||
json.dumps(project_data, ensure_ascii=False)),
|
||
)
|
||
project_id = cur.fetchone()['id']
|
||
conn.commit()
|
||
except Exception as e:
|
||
return jsonify({'error': f'create project failed: {e}'}), 500
|
||
|
||
# Обновляем roblox_imports
|
||
try:
|
||
with pg_conn() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"UPDATE roblox_imports SET project_id=%s, status='done', "
|
||
"finished_at=NOW(), assets_failed=%s "
|
||
"WHERE rbxl_sha256=%s",
|
||
(project_id, failed, preview_hash),
|
||
)
|
||
conn.commit()
|
||
except Exception:
|
||
pass
|
||
|
||
return jsonify({
|
||
'project_id': project_id,
|
||
'redirect': f'/edit/{project_id}',
|
||
'assets_downloaded': len(asset_url_map),
|
||
'assets_failed': failed,
|
||
})
|
||
|
||
|
||
def _resolve_asset_urls(project_data: dict, asset_map: dict) -> None:
|
||
"""Walks project_data, заменяет {rbxAssetId: X} → url из asset_map."""
|
||
scene = project_data.get('scene', {})
|
||
for glb in scene.get('glbModels', []):
|
||
rid = glb.get('rbxAssetId')
|
||
if rid in asset_map:
|
||
glb['url'] = asset_map[rid]
|
||
for snd in scene.get('sounds', []):
|
||
rid = snd.get('rbxAssetId')
|
||
if rid in asset_map:
|
||
snd['url'] = asset_map[rid]
|
||
|
||
|
||
if __name__ == '__main__':
|
||
app.run(host='0.0.0.0', port=8690, debug=False)
|