2f9d6a21f6
Фронт студии (studio.rublox.pro) делал POST /import/rbxl/analyze на minecraftia-school.ru/api-rbxl, preflight (OPTIONS) не получал Access-Control-Allow-Origin → CORS ошибка. Фиксы: - after_request гарантированно ставит CORS-заголовки на ВСЕ ответы (включая OPTIONS) — раньше flask-cors иногда их не отдавал - Явный handler для OPTIONS /import/rbxl/analyze + create - Headers: Allow-Origin=*, Allow-Methods, Allow-Headers content-type+x-user-id - Убрал ALLOWED_USER_IDS=[1] (импорт открыт всем — кнопка в UI уже без гейтинга, см. вики «Импорт из Roblox») Деплой: вручную через SSH на VM 130 (rbxl-importer не имеет CI/CD).
436 lines
18 KiB
Python
436 lines
18 KiB
Python
"""
|
||
app.py — Flask API rbxl-importer.
|
||
|
||
Endpoints:
|
||
POST /import/rbxl/analyze
|
||
body: multipart, file=<.rbxl>
|
||
resp: {
|
||
"preview_hash": str, # для следующего шага
|
||
"report": {
|
||
"filename": str, "size_bytes": int, "version": int,
|
||
"class_count": int, "instance_count": int,
|
||
"top_classes": [{"class": str, "count": int}, ...],
|
||
"scripts_total": int,
|
||
"assets_to_download": int,
|
||
"warnings": [str, ...]
|
||
}
|
||
}
|
||
|
||
POST /import/rbxl/create
|
||
body: { "preview_hash": str, "title": str, "auth_user_id": int }
|
||
resp: { "project_id": int, "redirect": "/edit/<id>" }
|
||
— Парсит → конвертит → скачивает ассеты → создаёт запись в kubikon3d_projects.
|
||
|
||
GET /health
|
||
resp: { ok: true, version: "0.1.0" }
|
||
|
||
Безопасность: эндпоинты доступны только МИНу (user_id=1).
|
||
Проверка через X-User-Id header (NPM прокинет после JWT-проверки в user-сервисе).
|
||
"""
|
||
import os
|
||
import sys
|
||
import json
|
||
import hashlib
|
||
import tempfile
|
||
import logging
|
||
from pathlib import Path
|
||
from flask import Flask, request, jsonify
|
||
from flask_cors import CORS
|
||
import psycopg2
|
||
from psycopg2.extras import RealDictCursor
|
||
import redis
|
||
|
||
sys.path.insert(0, os.path.dirname(__file__))
|
||
|
||
from rbxl_parser import parse, class_histogram
|
||
from converter import Converter
|
||
from asset_downloader import AssetDownloader, PendingDownload
|
||
from mesh_converter import parse_roblox_mesh, build_glb, MeshParseError
|
||
|
||
logging.basicConfig(level=logging.INFO)
|
||
logger = logging.getLogger('rbxl-importer')
|
||
|
||
PG_DSN = os.environ.get(
|
||
'PG_DSN',
|
||
'host=192.168.1.152 port=25435 user=min password=5cb157970ad5e3b2952ed05decaf1bab dbname=storys_db'
|
||
)
|
||
REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379/0')
|
||
STORAGE_ROOT = os.environ.get('STORAGE_ROOT', '/opt/roblox-assets')
|
||
PUBLIC_ASSET_BASE = os.environ.get('PUBLIC_ASSET_BASE', 'https://assets.rublox.pro/roblox')
|
||
|
||
MAX_RBXL_SIZE = 50 * 1024 * 1024 # 50 MB
|
||
|
||
app = Flask(__name__)
|
||
# CORS открыт для всех источников — фронт студии живёт на studio.rublox.pro,
|
||
# api-rbxl проксируется через NPM на minecraftia-school.ru/api-rbxl/*.
|
||
# Поддерживаем preflight (OPTIONS) явно через after_request — иногда
|
||
# flask-cors не отдавал заголовки для OPTIONS если NPM их перекрывал.
|
||
CORS(app, resources={r'/*': {'origins': '*'}}, supports_credentials=False)
|
||
|
||
|
||
@app.after_request
|
||
def _add_cors_headers(resp):
|
||
resp.headers['Access-Control-Allow-Origin'] = '*'
|
||
resp.headers['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS'
|
||
resp.headers['Access-Control-Allow-Headers'] = 'Content-Type, X-User-Id, X-User-Login'
|
||
resp.headers['Access-Control-Max-Age'] = '3600'
|
||
return resp
|
||
|
||
|
||
@app.route('/import/rbxl/analyze', methods=['OPTIONS'])
|
||
@app.route('/import/rbxl/create', methods=['OPTIONS'])
|
||
def _preflight():
|
||
return '', 204
|
||
|
||
# Devlog для удалённой отладки dev-сессий студии: фронт пушит сюда
|
||
# console.error/warn, failed network requests, неожиданные exceptions.
|
||
from app_devlog import devlog_bp
|
||
app.register_blueprint(devlog_bp)
|
||
|
||
try:
|
||
rds = redis.from_url(REDIS_URL, decode_responses=False)
|
||
rds.ping()
|
||
logger.info(f'Redis connected: {REDIS_URL}')
|
||
except Exception as e:
|
||
logger.warning(f'Redis NOT connected: {e}; preview cache отключён')
|
||
rds = None
|
||
|
||
|
||
def pg_conn():
|
||
return psycopg2.connect(PG_DSN)
|
||
|
||
|
||
def auth_check(req) -> int:
|
||
"""Возвращает user_id если ОК, иначе бросает RuntimeError."""
|
||
# X-User-Id выставляется upstream NPM или service-user после JWT-проверки.
|
||
# В dev можно через header X-Auth-Override (только в LAN).
|
||
user_id_str = req.headers.get('X-User-Id') or req.headers.get('X-Auth-Override')
|
||
if not user_id_str:
|
||
raise RuntimeError('No X-User-Id header')
|
||
try:
|
||
uid = int(user_id_str)
|
||
except ValueError:
|
||
raise RuntimeError(f'Bad X-User-Id: {user_id_str!r}')
|
||
# Импорт открыт всем (см. вики «Импорт из Roblox»).
|
||
return uid
|
||
|
||
|
||
@app.errorhandler(Exception)
|
||
def on_error(e):
|
||
logger.exception('handler error')
|
||
return jsonify({'error': str(e), 'type': type(e).__name__}), 500
|
||
|
||
|
||
@app.get('/health')
|
||
def health():
|
||
return jsonify({'ok': True, 'version': '0.1.0', 'service': 'rbxl-importer'})
|
||
|
||
|
||
@app.post('/import/rbxl/analyze')
|
||
def analyze():
|
||
try:
|
||
user_id = auth_check(request)
|
||
except RuntimeError as e:
|
||
return jsonify({'error': str(e)}), 403
|
||
|
||
if 'file' not in request.files:
|
||
return jsonify({'error': 'file field required'}), 400
|
||
upload = request.files['file']
|
||
blob = upload.read()
|
||
if len(blob) > MAX_RBXL_SIZE:
|
||
return jsonify({'error': f'file too large (> {MAX_RBXL_SIZE} bytes)'}), 413
|
||
# Авто-детект XML vs Binary формата.
|
||
# Бинарный: <roblox!\x89\xff\r\n\x1a\n (magic bytes).
|
||
# XML (старые карты до 2010): <roblox version="4">...
|
||
stripped = blob.lstrip()
|
||
is_binary = stripped.startswith(b'<roblox!')
|
||
is_xml = stripped.startswith(b'<roblox') and not is_binary
|
||
|
||
if not is_binary and not is_xml:
|
||
return jsonify({'error': 'not a .rbxl file (no <roblox magic)'}), 400
|
||
|
||
# Парсим
|
||
try:
|
||
if is_xml:
|
||
from rbxl_xml_parser import parse_xml
|
||
model = parse_xml(blob)
|
||
else:
|
||
model = parse(blob)
|
||
except Exception as e:
|
||
return jsonify({'error': f'parse failed: {e}'}), 422
|
||
|
||
# Конвертим (без скачки ассетов — просто узнаем сколько их)
|
||
conv = Converter(model)
|
||
project_data = conv.convert()
|
||
|
||
# Лог импорта
|
||
rbxl_sha = hashlib.sha256(blob).hexdigest()
|
||
asset_ids = list(set(conv.stats.asset_ids_needed))
|
||
|
||
# Кладём blob во временный файл + сохраняем sha256 как preview_hash
|
||
preview_hash = rbxl_sha
|
||
if rds:
|
||
rds.setex(f'rbxl:blob:{preview_hash}', 1200, blob) # 20 минут
|
||
rds.setex(f'rbxl:project:{preview_hash}', 1200,
|
||
json.dumps(project_data).encode('utf-8'))
|
||
rds.setex(f'rbxl:assets:{preview_hash}', 1200,
|
||
json.dumps(asset_ids).encode('utf-8'))
|
||
|
||
# Отчёт
|
||
hist = class_histogram(model)
|
||
top = sorted(hist.items(), key=lambda x: -x[1])[:25]
|
||
report = {
|
||
'filename': upload.filename or 'unknown.rbxl',
|
||
'size_bytes': len(blob),
|
||
'version': model.version,
|
||
'class_count': model.class_count,
|
||
'instance_count': model.instance_count,
|
||
'top_classes': [{'class': c, 'count': n} for c, n in top],
|
||
'primitives_created': conv.stats.primitives_created,
|
||
'glb_models_created': conv.stats.glb_models_created,
|
||
'scripts_total': conv.stats.scripts_collected,
|
||
'scripts_skipped': conv.stats.scripts_skipped,
|
||
'parts_dropped': conv.stats.parts_dropped,
|
||
'assets_to_download': len(asset_ids),
|
||
'warnings': conv.stats.warnings[:30],
|
||
}
|
||
|
||
# Запись в roblox_imports (status='analyzed')
|
||
try:
|
||
with pg_conn() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"INSERT INTO roblox_imports "
|
||
"(user_id, rbxl_filename, rbxl_size, rbxl_sha256, "
|
||
" instance_count, class_count, assets_total, status) "
|
||
"VALUES (%s, %s, %s, %s, %s, %s, %s, 'analyzed') "
|
||
"RETURNING id",
|
||
(user_id, report['filename'], report['size_bytes'], rbxl_sha,
|
||
report['instance_count'], report['class_count'], report['assets_to_download']),
|
||
)
|
||
import_id = cur.fetchone()[0]
|
||
conn.commit()
|
||
report['import_id'] = import_id
|
||
except Exception as e:
|
||
logger.warning(f'roblox_imports insert failed: {e}')
|
||
|
||
return jsonify({'preview_hash': preview_hash, 'report': report})
|
||
|
||
|
||
@app.post('/import/rbxl/create')
|
||
def create():
|
||
try:
|
||
user_id = auth_check(request)
|
||
except RuntimeError as e:
|
||
return jsonify({'error': str(e)}), 403
|
||
|
||
data = request.get_json(silent=True) or {}
|
||
preview_hash = data.get('preview_hash')
|
||
title = (data.get('title') or '').strip() or 'Импортировано из Roblox'
|
||
# scripts_mode: 'disabled' (default) — оставить в проекте, но enabled=False
|
||
# 'enabled' — попытаться запустить, может вешать
|
||
# 'skip' — не импортировать совсем
|
||
scripts_mode = data.get('scripts_mode', 'disabled')
|
||
if scripts_mode not in ('disabled', 'enabled', 'skip'):
|
||
scripts_mode = 'disabled'
|
||
# gui_mode: 'all' / 'screen-only' (только ScreenGui-HUD) / 'skip' (без GUI)
|
||
gui_mode = data.get('gui_mode', 'all')
|
||
if gui_mode not in ('all', 'screen-only', 'skip'):
|
||
gui_mode = 'all'
|
||
|
||
if not preview_hash:
|
||
return jsonify({'error': 'preview_hash required'}), 400
|
||
|
||
if not rds:
|
||
return jsonify({'error': 'redis unavailable, cannot retrieve preview'}), 503
|
||
|
||
blob_cached = rds.get(f'rbxl:blob:{preview_hash}')
|
||
project_cached = rds.get(f'rbxl:project:{preview_hash}')
|
||
if not project_cached:
|
||
return jsonify({'error': 'preview expired, please re-analyze'}), 410
|
||
|
||
project_data = json.loads(project_cached.decode('utf-8'))
|
||
asset_ids_raw = rds.get(f'rbxl:assets:{preview_hash}')
|
||
asset_ids = json.loads(asset_ids_raw.decode('utf-8')) if asset_ids_raw else []
|
||
|
||
# Скачиваем ассеты
|
||
dl = AssetDownloader(db_dsn=PG_DSN, storage_root=STORAGE_ROOT,
|
||
public_base=PUBLIC_ASSET_BASE)
|
||
asset_url_map = {} # rbx_id → public_url
|
||
failed = 0
|
||
for rbx_id in asset_ids:
|
||
try:
|
||
rec = dl.fetch_sync(int(rbx_id))
|
||
# Конвертим mesh → glb если это mesh
|
||
if rec.asset_kind == 'mesh' and not rec.converted_path:
|
||
try:
|
||
with open(rec.raw_path, 'rb') as f:
|
||
mesh_blob = f.read()
|
||
mesh = parse_roblox_mesh(mesh_blob)
|
||
glb = build_glb(mesh)
|
||
sha = hashlib.sha256(glb).hexdigest()
|
||
glb_dir = os.path.join(STORAGE_ROOT, 'converted', sha[:2])
|
||
os.makedirs(glb_dir, exist_ok=True)
|
||
glb_path = os.path.join(glb_dir, f'{sha}.glb')
|
||
if not os.path.exists(glb_path):
|
||
with open(glb_path, 'wb') as f:
|
||
f.write(glb)
|
||
public_glb = f'{PUBLIC_ASSET_BASE}/converted/{sha[:2]}/{sha}.glb'
|
||
asset_url_map[int(rbx_id)] = public_glb
|
||
# Обновляем БД
|
||
with pg_conn() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"UPDATE roblox_assets SET converted_path=%s, "
|
||
"converted_sha256=%s, converted_size_bytes=%s "
|
||
"WHERE rbx_asset_id=%s",
|
||
(glb_path, sha, len(glb), int(rbx_id)),
|
||
)
|
||
conn.commit()
|
||
except (MeshParseError, Exception) as me:
|
||
logger.warning(f'mesh→glb failed for {rbx_id}: {me}')
|
||
asset_url_map[int(rbx_id)] = rec.public_url
|
||
else:
|
||
asset_url_map[int(rbx_id)] = rec.public_url
|
||
except PendingDownload:
|
||
failed += 1
|
||
except Exception as e:
|
||
failed += 1
|
||
logger.warning(f'asset {rbx_id} download failed: {e}')
|
||
|
||
# Подставляем URLs в project_data
|
||
_resolve_asset_urls(project_data, asset_url_map)
|
||
|
||
# Применяем scripts_mode: меняем поле enabled в метадате каждого скрипта
|
||
# либо удаляем все скрипты полностью.
|
||
_apply_scripts_mode(project_data, scripts_mode)
|
||
|
||
# Применяем gui_mode: удаляем 3D-GUI (BillboardGui/SurfaceGui) или вообще
|
||
# всё, если выбрано 'skip'.
|
||
_apply_gui_mode(project_data, gui_mode)
|
||
|
||
# Создаём проект в kubikon3d_projects
|
||
# Используем bridge на user-service / storys API, ИЛИ напрямую в storys_db.
|
||
# Прямой INSERT — проще для MVP. id автогенерируется.
|
||
try:
|
||
with pg_conn() as conn:
|
||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||
# Возьмём следующий id из последовательности
|
||
cur.execute(
|
||
"INSERT INTO kubikon3d_projects "
|
||
"(user_id, title, description, project_data, "
|
||
" status, is_test, is_public, is_published, "
|
||
" feed_eligible, feed_blocked_forever, quality_state, "
|
||
" created_at, updated_at) "
|
||
"VALUES (%s, %s, %s, %s, "
|
||
" 'draft', true, false, false, "
|
||
" false, false, 'new', NOW(), NOW()) "
|
||
"RETURNING id",
|
||
(user_id, title,
|
||
f'Импортировано из .rbxl (rbxl-importer v0.1.0)',
|
||
json.dumps(project_data, ensure_ascii=False)),
|
||
)
|
||
project_id = cur.fetchone()['id']
|
||
conn.commit()
|
||
except Exception as e:
|
||
return jsonify({'error': f'create project failed: {e}'}), 500
|
||
|
||
# Обновляем roblox_imports
|
||
try:
|
||
with pg_conn() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"UPDATE roblox_imports SET project_id=%s, status='done', "
|
||
"finished_at=NOW(), assets_failed=%s "
|
||
"WHERE rbxl_sha256=%s",
|
||
(project_id, failed, preview_hash),
|
||
)
|
||
conn.commit()
|
||
except Exception:
|
||
pass
|
||
|
||
return jsonify({
|
||
'project_id': project_id,
|
||
'redirect': f'/edit/{project_id}',
|
||
'assets_downloaded': len(asset_url_map),
|
||
'assets_failed': failed,
|
||
})
|
||
|
||
|
||
def _resolve_asset_urls(project_data: dict, asset_map: dict) -> None:
|
||
"""Walks project_data, заменяет {rbxAssetId: X} → url из asset_map."""
|
||
scene = project_data.get('scene', {})
|
||
for glb in scene.get('glbModels', []):
|
||
rid = glb.get('rbxAssetId')
|
||
if rid in asset_map:
|
||
glb['url'] = asset_map[rid]
|
||
for snd in scene.get('sounds', []):
|
||
rid = snd.get('rbxAssetId')
|
||
if rid in asset_map:
|
||
snd['url'] = asset_map[rid]
|
||
|
||
|
||
def _apply_gui_mode(project_data: dict, mode: str) -> None:
|
||
"""Фильтрует scene.gui[] по режиму.
|
||
|
||
'all' — оставить всё (default).
|
||
'screen-only' — оставить только ScreenGui-HUD, удалить billboard/surface.
|
||
Карты с 200+ BillboardGui (Robloxity) перестают тормозить.
|
||
'skip' — удалить gui[] совсем.
|
||
"""
|
||
scene = project_data.get('scene', {})
|
||
if mode == 'skip':
|
||
scene['gui'] = []
|
||
return
|
||
if mode == 'screen-only':
|
||
gui = scene.get('gui', [])
|
||
scene['gui'] = [g for g in gui
|
||
if g.get('gui_container_kind', 'screen') == 'screen']
|
||
return
|
||
# 'all' — без изменений
|
||
|
||
|
||
def _apply_scripts_mode(project_data: dict, mode: str) -> None:
|
||
"""Применяет режим scripts_mode к проекту.
|
||
|
||
mode='disabled' (default): для каждого скрипта меняем JSON-метадату
|
||
на 2-й строке packed-кода — выставляем enabled=False. GameRuntime
|
||
уже умеет уважать этот флаг и не запускает.
|
||
mode='enabled': оставляем как было (как пришло из конвертера).
|
||
mode='skip': удаляем все scripts из scene.scripts полностью.
|
||
"""
|
||
scene = project_data.get('scene', {})
|
||
scripts = scene.get('scripts', [])
|
||
if not scripts:
|
||
return
|
||
|
||
if mode == 'skip':
|
||
scene['scripts'] = []
|
||
return
|
||
|
||
if mode == 'enabled':
|
||
return # ничего не делаем
|
||
|
||
# mode == 'disabled' — патчим метадату каждого скрипта.
|
||
# Формат packed-кода (см. converter._convert_script):
|
||
# "// @roblox-lua\n// {JSON}\n/* lua_source:\n...source...\n*/\n"
|
||
for s in scripts:
|
||
code = s.get('code', '')
|
||
lines = code.split('\n', 2)
|
||
if len(lines) < 2 or not lines[0].startswith('// @roblox-lua'):
|
||
continue
|
||
meta_line = lines[1]
|
||
if not meta_line.startswith('// '):
|
||
continue
|
||
try:
|
||
meta = json.loads(meta_line[3:])
|
||
meta['enabled'] = False
|
||
new_meta_line = '// ' + json.dumps(meta, ensure_ascii=False)
|
||
s['code'] = lines[0] + '\n' + new_meta_line + '\n' + (lines[2] if len(lines) > 2 else '')
|
||
except (json.JSONDecodeError, ValueError):
|
||
continue
|
||
|
||
|
||
if __name__ == '__main__':
|
||
app.run(host='0.0.0.0', port=8690, debug=False)
|